diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 9cf6a18d82..d6297f0781 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -836,6 +836,9 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByDataSourceAndComponentId(data func (impl CiArtifactRepositoryImpl) FindCiArtifactByImagePaths(images []string) ([]CiArtifact, error) { var ciArtifacts []CiArtifact + if len(images) == 0 { + return nil, nil + } err := impl.dbConnection. Model(&ciArtifacts). Where(" image in (?) ", pg.In(images)). diff --git a/internal/sql/repository/CustomTagRepository.go b/internal/sql/repository/CustomTagRepository.go index 4b33725afc..0eb41620f8 100644 --- a/internal/sql/repository/CustomTagRepository.go +++ b/internal/sql/repository/CustomTagRepository.go @@ -56,6 +56,7 @@ type ImageTagRepository interface { DeactivateImagePathReservationByImagePaths(tx *pg.Tx, imagePaths []string) error DeactivateImagePathReservationByImagePathReservationIds(tx *pg.Tx, imagePathReservationIds []int) error DisableCustomTag(entityKey int, entityValue string) error + GetImagePathsByIds(ids []int) ([]*ImagePathReservation, error) } type ImageTagRepositoryImpl struct { @@ -139,6 +140,9 @@ func (impl *ImageTagRepositoryImpl) InsertImagePath(tx *pg.Tx, reservation *Imag } func (impl *ImageTagRepositoryImpl) DeactivateImagePathReservationByImagePaths(tx *pg.Tx, imagePaths []string) error { + if len(imagePaths) == 0 { + return nil + } query := `UPDATE image_path_reservation set active=false where image_path in (?)` _, err := tx.Exec(query, pg.In(imagePaths)) if err != nil && err != pg.ErrNoRows { @@ -161,3 +165,13 @@ func (impl *ImageTagRepositoryImpl) DisableCustomTag(entityKey int, entityValue _, err := impl.dbConnection.Exec(query, entityKey, entityValue) return err } +func (impl *ImageTagRepositoryImpl) GetImagePathsByIds(ids []int) ([]*ImagePathReservation, error) { + var imagePaths []*ImagePathReservation + if len(ids) == 0 { + return imagePaths, nil + } + err := impl.dbConnection.Model(&imagePaths). + Where("id in (?) ", pg.In(ids)). + Where("active = ?", true).Select() + return imagePaths, err +} diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go index 413e51b5fc..26a20f2843 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -92,7 +92,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) er cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE // handling plugin specific logic - pluginImagePathReservationIds, err := impl.SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest, pipeline.Id, types.POST, cdWf.CiArtifact) + pluginImagePathReservationIds, err := impl.setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest, pipeline.Id, types.POST, cdWf.CiArtifact) if err != nil { runner.Status = pipelineConfig.WorkflowFailed runner.Message = err.Error() diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 37596e7479..244c5b0c47 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -39,6 +39,7 @@ import ( repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" + bean3 "github.com/devtron-labs/devtron/pkg/plugin/bean" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/sql" util3 "github.com/devtron-labs/devtron/pkg/util" @@ -110,7 +111,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) err } cdStageWorkflowRequest.StageType = types.PRE // handling copyContainerImage plugin specific logic - imagePathReservationIds, err := impl.SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest, pipeline.Id, types.PRE, artifact) + imagePathReservationIds, err := impl.setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest, pipeline.Id, types.PRE, artifact) if err != nil { runner.Status = pipelineConfig.WorkflowFailed runner.Message = err.Error() @@ -236,95 +237,121 @@ func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx co return nil } -func (impl *TriggerServiceImpl) SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { - copyContainerImagePluginId, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(pipeline.COPY_CONTAINER_IMAGE) - var imagePathReservationIds []int +// setCopyContainerImagePluginDataAndReserveImages sets required fields in cdStageWorkflowRequest and reserve images generated by plugin +func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { + + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(pipeline.COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting copyContainerImage plugin id", "err", err) - return imagePathReservationIds, err + return nil, err } - for _, step := range cdStageWorkflowRequest.PrePostDeploySteps { - if copyContainerImagePluginId != 0 && step.RefPluginId == copyContainerImagePluginId { - var pipelineStageEntityType int - if pipelineStage == types.PRE { - pipelineStageEntityType = pipelineConfigBean.EntityTypePreCD - } else { - pipelineStageEntityType = pipelineConfigBean.EntityTypePostCD - } - customTagId := -1 - var DockerImageTag string - customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineStageEntityType, strconv.Itoa(pipelineId)) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching custom tag data", "err", err) - return imagePathReservationIds, err - } + pluginIdToVersionMap := make(map[int]string) + for _, p := range copyContainerImagePluginDetail { + pluginIdToVersionMap[p.Id] = p.Version + } - if !customTag.Enabled { - // case when custom tag is not configured - source image tag will be taken as docker image tag - pluginTriggerImageSplit := strings.Split(artifact.Image, ":") - DockerImageTag = pluginTriggerImageSplit[len(pluginTriggerImageSplit)-1] - } else { - // for copyContainerImage plugin parse destination images and save its data in image path reservation table - customTagDbObject, customDockerImageTag, err := impl.customTagService.GetCustomTag(pipelineStageEntityType, strconv.Itoa(pipelineId)) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching custom tag by entity key and value for CD", "err", err) - return imagePathReservationIds, err - } - if customTagDbObject != nil && customTagDbObject.Id > 0 { - customTagId = customTagDbObject.Id - } - DockerImageTag = customDockerImageTag - } + dockerImageTag, customTagId, err := impl.getDockerTagAndCustomTagIdForPlugin(pipelineStage, pipelineId, artifact) + if err != nil { + impl.logger.Errorw("error in getting docker tag", "err", err) + return nil, err + } - var sourceDockerRegistryId string - if artifact.DataSource == repository.PRE_CD || artifact.DataSource == repository.POST_CD || artifact.DataSource == repository.POST_CI { - if artifact.CredentialsSourceType == repository.GLOBAL_CONTAINER_REGISTRY { - sourceDockerRegistryId = artifact.CredentialSourceValue - } - } else { - sourceDockerRegistryId = cdStageWorkflowRequest.DockerRegistryId - } - registryDestinationImageMap, registryCredentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, DockerImageTag, cdStageWorkflowRequest.CiArtifactDTO.Image, sourceDockerRegistryId) + var sourceDockerRegistryId string + if artifact.DataSource == repository.PRE_CD || artifact.DataSource == repository.POST_CD || artifact.DataSource == repository.POST_CI { + if artifact.CredentialsSourceType == repository.GLOBAL_CONTAINER_REGISTRY { + sourceDockerRegistryId = artifact.CredentialSourceValue + } + } else { + sourceDockerRegistryId = cdStageWorkflowRequest.DockerRegistryId + } + + registryCredentialMap := make(map[string]bean3.RegistryCredentials) + var allDestinationImages []string //saving all images to be reserved in this array + + for _, step := range cdStageWorkflowRequest.PrePostDeploySteps { + if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + registryDestinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, dockerImageTag, cdStageWorkflowRequest.CiArtifactDTO.Image, sourceDockerRegistryId) if err != nil { impl.logger.Errorw("error in parsing copyContainerImage input variable", "err", err) - return imagePathReservationIds, err - } - var destinationImages []string - for _, images := range registryDestinationImageMap { - for _, image := range images { - destinationImages = append(destinationImages, image) - } - } - // fetch already saved artifacts to check if they are already present - savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(destinationImages) - if err != nil { - impl.logger.Errorw("error in fetching artifacts by image path", "err", err) - return imagePathReservationIds, err + return nil, err } - if len(savedCIArtifacts) > 0 { - // if already present in ci artifact, return "image path already in use error" - return imagePathReservationIds, pipelineConfigBean.ErrImagePathInUse + if version == pipeline.COPY_CONTAINER_IMAGE_VERSION_V1 { + // this is needed in ci runner only for v1 + cdStageWorkflowRequest.RegistryDestinationImageMap = registryDestinationImageMap } - imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, registryDestinationImageMap) - if err != nil { - impl.logger.Errorw("error in reserving image", "err", err) - return imagePathReservationIds, err + for _, images := range registryDestinationImageMap { + allDestinationImages = append(allDestinationImages, images...) } - cdStageWorkflowRequest.RegistryDestinationImageMap = registryDestinationImageMap - cdStageWorkflowRequest.RegistryCredentialMap = registryCredentialMap - var pluginArtifactStage string - if pipelineStage == types.PRE { - pluginArtifactStage = repository.PRE_CD - } else { - pluginArtifactStage = repository.POST_CD + for k, v := range credentialMap { + registryCredentialMap[k] = v } - cdStageWorkflowRequest.PluginArtifactStage = pluginArtifactStage } } + + // set data in cdStageWorkflowRequest needed for copy container image plugin + + cdStageWorkflowRequest.RegistryCredentialMap = registryCredentialMap + cdStageWorkflowRequest.DockerImageTag = dockerImageTag + if pipelineStage == types.PRE { + cdStageWorkflowRequest.PluginArtifactStage = repository.PRE_CD + } else { + cdStageWorkflowRequest.PluginArtifactStage = repository.POST_CD + } + + // fetch already saved artifacts to check if they are already present + + savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) + if err != nil { + impl.logger.Errorw("error in fetching artifacts by image path", "err", err) + return nil, err + } + if len(savedCIArtifacts) > 0 { + // if already present in ci artifact, return "image path already in use error" + return nil, pipelineConfigBean.ErrImagePathInUse + } + // reserve all images where data will be + imagePathReservationIds, err := impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) + if err != nil { + impl.logger.Errorw("error in reserving image", "err", err) + return imagePathReservationIds, err + } return imagePathReservationIds, nil } +func (impl *TriggerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStage string, pipelineId int, artifact *repository.CiArtifact) (string, int, error) { + var pipelineStageEntityType int + if pipelineStage == types.PRE { + pipelineStageEntityType = pipelineConfigBean.EntityTypePreCD + } else { + pipelineStageEntityType = pipelineConfigBean.EntityTypePostCD + } + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineStageEntityType, strconv.Itoa(pipelineId)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom tag data", "err", err) + return "", 0, err + } + var DockerImageTag string + customTagId := -1 // if customTag is not configured id=-1 will be saved in image_path_reservation table for image reservation + if !customTag.Enabled { + // case when custom tag is not configured - source image tag will be taken as docker image tag + pluginTriggerImageSplit := strings.Split(artifact.Image, ":") + DockerImageTag = pluginTriggerImageSplit[len(pluginTriggerImageSplit)-1] + } else { + // for copyContainerImage plugin parse destination images and save its data in image path reservation table + customTagDbObject, customDockerImageTag, err := impl.customTagService.GetCustomTag(pipelineStageEntityType, strconv.Itoa(pipelineId)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom tag by entity key and value for CD", "err", err) + return "", 0, err + } + if customTagDbObject != nil && customTagDbObject.Id > 0 { + customTagId = customTagDbObject.Id + } + DockerImageTag = customDockerImageTag + } + return DockerImageTag, customTagId, nil +} + func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { if cdPipeline.App.Id == 0 { appModel, err := impl.appRepository.FindById(cdPipeline.AppId) @@ -843,20 +870,20 @@ func (impl *TriggerServiceImpl) getSourceCiPipelineForArtifact(ciPipeline pipeli return sourceCiPipeline, nil } -func (impl *TriggerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, registryImageMap map[string][]string) ([]int, error) { +func (impl *TriggerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { var imagePathReservationIds []int - for _, images := range registryImageMap { - for _, image := range images { - imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) - if err != nil { - impl.logger.Errorw("Error in marking custom tag reserved", "err", err) - return imagePathReservationIds, err - } - if imagePathReservationData != nil { - imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) - } + + for _, image := range destinationImages { + imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) + if err != nil { + impl.logger.Errorw("Error in marking custom tag reserved", "err", err) + return imagePathReservationIds, err + } + if imagePathReservationData != nil { + imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) } } + return imagePathReservationIds, nil } diff --git a/pkg/eventProcessor/bean/pluginArtifactsBean.go b/pkg/eventProcessor/bean/pluginArtifactsBean.go new file mode 100644 index 0000000000..4c60cea04c --- /dev/null +++ b/pkg/eventProcessor/bean/pluginArtifactsBean.go @@ -0,0 +1,61 @@ +package bean + +import ( + "slices" + "time" +) + +type Kind string +type CredentialSourceType string +type ArtifactType string + +const ( + PluginArtifactsKind Kind = "PluginArtifacts" + GlobalContainerRegistrySource CredentialSourceType = "global_container_registry" + ArtifactTypeContainer ArtifactType = "CONTAINER" +) + +type PluginArtifacts struct { + Kind Kind `json:"Kind"` + Artifacts []Artifact `json:"Artifacts"` +} + +func NewPluginArtifact() *PluginArtifacts { + return &PluginArtifacts{ + Kind: PluginArtifactsKind, + Artifacts: make([]Artifact, 0), + } +} + +func (p *PluginArtifacts) MergePluginArtifact(pluginArtifact *PluginArtifacts) { + if pluginArtifact == nil { + return + } + p.Artifacts = append(p.Artifacts, pluginArtifact.Artifacts...) +} + +func (p *PluginArtifacts) GetRegistryToUniqueContainerArtifactDataMapping() map[string][]string { + registryToImageMapping := make(map[string][]string) + for _, artifact := range p.Artifacts { + if artifact.Type == ArtifactTypeContainer { + if artifact.CredentialsSourceType == GlobalContainerRegistrySource { + if _, ok := registryToImageMapping[artifact.CredentialSourceValue]; !ok { + registryToImageMapping[artifact.CredentialSourceValue] = make([]string, 0) + } + registryToImageMapping[artifact.CredentialSourceValue] = append(registryToImageMapping[artifact.CredentialSourceValue], artifact.Data...) + slices.Sort(registryToImageMapping[artifact.CredentialSourceValue]) + slices.Compact(registryToImageMapping[artifact.CredentialSourceValue]) + } + } + } + return registryToImageMapping +} + +type Artifact struct { + Type ArtifactType `json:"Type"` + Data []string `json:"Data"` + CredentialsSourceType CredentialSourceType `json:"CredentialsSourceType"` + CredentialSourceValue string `json:"CredentialSourceValue"` + CreatedByPluginIdentifier string `json:"createdByPluginIdentifier"` + CreatedOn time.Time `json:"createdOn"` +} diff --git a/pkg/eventProcessor/bean/workflowEventBean.go b/pkg/eventProcessor/bean/workflowEventBean.go index 755defdd3e..37112b5fa2 100644 --- a/pkg/eventProcessor/bean/workflowEventBean.go +++ b/pkg/eventProcessor/bean/workflowEventBean.go @@ -38,6 +38,7 @@ type CdStageCompleteEvent struct { PipelineName string `json:"pipelineName"` CiArtifactDTO pipelineConfig.CiArtifactDTO `json:"ciArtifactDTO"` PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` + PluginArtifacts *PluginArtifacts `json:"pluginArtifacts"` } type UserDeploymentRequest struct { @@ -81,6 +82,7 @@ type CiCompleteEvent struct { PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` PluginArtifactStage string `json:"pluginArtifactStage"` pluginImageDetails *registry.ImageDetailsFromCR + PluginArtifacts *PluginArtifacts `json:"pluginArtifacts"` } func (c *CiCompleteEvent) GetPluginImageDetails() *registry.ImageDetailsFromCR { diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index d01fb1bfe6..afd0318248 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -185,8 +185,15 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDStageCompleteEvent() error { return } } else if wfr.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST { + + pluginArtifacts := make(map[string][]string) + if cdStageCompleteEvent.PluginArtifacts != nil { + pluginArtifacts = cdStageCompleteEvent.PluginArtifacts.GetRegistryToUniqueContainerArtifactDataMapping() + globalUtil.MergeMaps(pluginArtifacts, cdStageCompleteEvent.PluginRegistryArtifactDetails) + } + impl.logger.Debugw("received post stage success event for workflow runner ", "wfId", strconv.Itoa(wfr.Id)) - err = impl.workflowDagExecutor.HandlePostStageSuccessEvent(triggerContext, wfr.CdWorkflowId, cdStageCompleteEvent.CdPipelineId, cdStageCompleteEvent.TriggeredBy, cdStageCompleteEvent.PluginRegistryArtifactDetails) + err = impl.workflowDagExecutor.HandlePostStageSuccessEvent(triggerContext, wfr, wfr.CdWorkflowId, cdStageCompleteEvent.CdPipelineId, cdStageCompleteEvent.TriggeredBy, pluginArtifacts) if err != nil { impl.logger.Errorw("deployment success event error", "err", err) return @@ -636,6 +643,12 @@ func (impl *WorkflowEventProcessorImpl) BuildCiArtifactRequest(event bean.CiComp event.TriggeredBy = 1 // system triggered event } + pluginArtifacts := make(map[string][]string) + if event.PluginArtifacts != nil { + pluginArtifacts = event.PluginArtifacts.GetRegistryToUniqueContainerArtifactDataMapping() + globalUtil.MergeMaps(pluginArtifacts, event.PluginRegistryArtifactDetails) + } + request := &wrokflowDagBean.CiArtifactWebhookRequest{ Image: event.DockerImage, ImageDigest: event.Digest, @@ -645,7 +658,7 @@ func (impl *WorkflowEventProcessorImpl) BuildCiArtifactRequest(event bean.CiComp UserId: event.TriggeredBy, WorkflowId: event.WorkflowId, IsArtifactUploaded: event.IsArtifactUploaded, - PluginRegistryArtifactDetails: event.PluginRegistryArtifactDetails, + PluginRegistryArtifactDetails: pluginArtifacts, PluginArtifactStage: event.PluginArtifactStage, } // if DataSource is empty, repository.WEBHOOK is considered as default diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index c9e0491e27..33546a0b1c 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -695,10 +695,10 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } // copyContainerImage plugin specific logic - var registryDestinationImageMap map[string][]string var registryCredentialMap map[string]bean2.RegistryCredentials var pluginArtifactStage string var imageReservationIds []int + var registryDestinationImageMap map[string][]string if !isJob { registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, fmt.Sprintf(pipelineConfigBean.ImagePathPattern, @@ -833,9 +833,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. if ciWorkflowConfig.LogsBucket == "" { ciWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() } - if len(registryDestinationImageMap) > 0 { - workflowRequest.PushImageBeforePostCI = true - } + switch workflowRequest.CloudProvider { case types.BLOB_STORAGE_S3: // No AccessKey is used for uploading artifacts, instead IAM based auth is used @@ -894,58 +892,81 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } func (impl *CiServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { - var registryDestinationImageMap map[string][]string - var registryCredentialMap map[string]bean2.RegistryCredentials - var pluginArtifactStage string - var imagePathReservationIds []int - copyContainerImagePluginId, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(COPY_CONTAINER_IMAGE) + + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) - return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, err + return nil, nil, "", nil, err + } + + pluginIdToVersionMap := make(map[int]string) + for _, p := range copyContainerImagePluginDetail { + pluginIdToVersionMap[p.Id] = p.Version } + for _, step := range preCiSteps { - if copyContainerImagePluginId != 0 && step.RefPluginId == copyContainerImagePluginId { + if _, ok := pluginIdToVersionMap[step.RefPluginId]; ok { // for copyContainerImage plugin parse destination images and save its data in image path reservation table - return nil, nil, pluginArtifactStage, nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") + return nil, nil, "", nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") } } + + registryCredentialMap := make(map[string]bean2.RegistryCredentials) + registryDestinationImageMap := make(map[string][]string) + var allDestinationImages []string //saving all images to be reserved in this array + for _, step := range postCiSteps { - if copyContainerImagePluginId != 0 && step.RefPluginId == copyContainerImagePluginId { - // for copyContainerImage plugin parse destination images and save its data in image path reservation table - registryDestinationImageMap, registryCredentialMap, err = impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) + if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + destinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) if err != nil { impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) - return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, err + return nil, nil, "", nil, err + } + if version == COPY_CONTAINER_IMAGE_VERSION_V1 { + // this is needed in ci runner only for v1 + registryDestinationImageMap = destinationImageMap + } + for _, images := range destinationImageMap { + allDestinationImages = append(allDestinationImages, images...) + } + for k, v := range credentialMap { + registryCredentialMap[k] = v } - pluginArtifactStage = repository5.POST_CI } } - for _, images := range registryDestinationImageMap { - for _, image := range images { - if image == buildImagePath { - return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, - pipelineConfigBean.ErrImagePathInUse - } + + pluginArtifactStage := repository5.POST_CI + for _, image := range allDestinationImages { + if image == buildImagePath { + return nil, registryCredentialMap, pluginArtifactStage, nil, + pipelineConfigBean.ErrImagePathInUse } } - imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, registryDestinationImageMap) + savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) + if err != nil { + impl.Logger.Errorw("error in fetching artifacts by image path", "err", err) + return nil, nil, pluginArtifactStage, nil, err + } + if len(savedCIArtifacts) > 0 { + // if already present in ci artifact, return "image path already in use error" + return nil, nil, pluginArtifactStage, nil, pipelineConfigBean.ErrImagePathInUse + } + imagePathReservationIds, err := impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) if err != nil { return nil, nil, pluginArtifactStage, imagePathReservationIds, err } return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil } -func (impl *CiServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, registryImageMap map[string][]string) ([]int, error) { +func (impl *CiServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { var imagePathReservationIds []int - for _, images := range registryImageMap { - for _, image := range images { - imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) - if err != nil { - impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) - return imagePathReservationIds, err - } - imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) + for _, image := range destinationImages { + imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) + if err != nil { + impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) + return imagePathReservationIds, err } + imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) } return imagePathReservationIds, nil } diff --git a/pkg/pipeline/CustomTagService.go b/pkg/pipeline/CustomTagService.go index 9869f70786..a17bd8ef5a 100644 --- a/pkg/pipeline/CustomTagService.go +++ b/pkg/pipeline/CustomTagService.go @@ -41,6 +41,7 @@ type CustomTagService interface { DeactivateImagePathReservationByImagePath(imagePaths []string) error DeactivateImagePathReservationByImageIds(imagePathReservationIds []int) error DisableCustomTagIfExist(tag bean.CustomTag) error + GetImagePathsByIds(ids []int) ([]*repository.ImagePathReservation, error) } type CustomTagServiceImpl struct { @@ -303,3 +304,7 @@ func (impl *CustomTagServiceImpl) DeactivateImagePathReservationByImageIds(image func (impl *CustomTagServiceImpl) DisableCustomTagIfExist(tag bean.CustomTag) error { return impl.customTagRepository.DisableCustomTag(tag.EntityKey, tag.EntityValue) } + +func (impl *CustomTagServiceImpl) GetImagePathsByIds(ids []int) ([]*repository.ImagePathReservation, error) { + return impl.customTagRepository.GetImagePathsByIds(ids) +} diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index d4e7e7cfea..847355dda7 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -2078,7 +2078,7 @@ func (impl *PipelineStageServiceImpl) BuildPluginVariableAndConditionDataForWfRe variableData.VariableType = bean.VARIABLE_TYPE_VALUE } else if variable.ValueType == repository2.PLUGIN_VARIABLE_VALUE_TYPE_GLOBAL { variableData.VariableType = bean.VARIABLE_TYPE_REF_GLOBAL - } else if variable.ValueType == repository2.PLUGIN_VARIABLE_VALUE_TYPE_PREVIOUS { + } else if variable.ValueType == repository2.PLUGIN_VARIABLE_VALUE_TYPE_PREVIOUS && !variable.IsExposed { variableData.VariableType = bean.VARIABLE_TYPE_REF_PLUGIN } if variable.VariableType == repository2.PLUGIN_VARIABLE_TYPE_INPUT { diff --git a/pkg/pipeline/pipelineStageVariableParser.go b/pkg/pipeline/pipelineStageVariableParser.go index a4497724c7..3dc9a9150d 100644 --- a/pkg/pipeline/pipelineStageVariableParser.go +++ b/pkg/pipeline/pipelineStageVariableParser.go @@ -33,8 +33,10 @@ type copyContainerImagePluginInputVariable = string type RefPluginName = string const ( - COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" - EMPTY_STRING = " " + COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" + COPY_CONTAINER_IMAGE_VERSION_V1 = "v1.0.0" + COPY_CONTAINER_IMAGE_VERSION_V2 = "v1.1.0" + EMPTY_STRING = " " ) const ( diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 5423ab22b2..a39cef039b 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -69,7 +69,7 @@ type GlobalPluginService interface { GetAllGlobalVariables(appType helper.AppType) ([]*GlobalVariable, error) ListAllPlugins(stageTypeReq string) ([]*bean2.PluginListComponentDto, error) GetPluginDetailById(pluginId int) (*bean2.PluginDetailDto, error) - GetRefPluginIdByRefPluginName(pluginName string) (refPluginId int, err error) + GetRefPluginIdByRefPluginName(pluginName string) (pluginVersionDetail []bean2.PluginsVersionDetail, err error) PatchPlugin(pluginDto *bean2.PluginMetadataDto, userId int32) (*bean2.PluginMetadataDto, error) GetDetailedPluginInfoByPluginId(pluginId int) (*bean2.PluginMetadataDto, error) GetAllDetailedPluginInfo() ([]*bean2.PluginMetadataDto, error) @@ -383,16 +383,23 @@ func getVariableDto(pluginVariable *repository.PluginStepVariable) *bean2.Plugin } } -func (impl *GlobalPluginServiceImpl) GetRefPluginIdByRefPluginName(pluginName string) (refPluginId int, err error) { +func (impl *GlobalPluginServiceImpl) GetRefPluginIdByRefPluginName(pluginName string) (pluginVersionDetail []bean2.PluginsVersionDetail, err error) { pluginMetadata, err := impl.globalPluginRepository.GetPluginByName(pluginName) if err != nil { impl.logger.Errorw("error in fetching plugin metadata by name", "err", err) - return 0, err + return nil, err } if pluginMetadata == nil { - return 0, nil + return nil, nil + } + pluginVersionDetail = make([]bean2.PluginsVersionDetail, 0) + for _, p := range pluginMetadata { + pluginVersionDetail = append(pluginVersionDetail, bean2.PluginsVersionDetail{ + PluginMetadataDto: &bean2.PluginMetadataDto{Id: p.Id}, + Version: p.PluginVersion, + }) } - return pluginMetadata[0].Id, nil + return pluginVersionDetail, nil } func (impl *GlobalPluginServiceImpl) PatchPlugin(pluginDto *bean2.PluginMetadataDto, userId int32) (*bean2.PluginMetadataDto, error) { diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 511f76279e..15a71c104b 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -43,6 +43,7 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/workflow/cd" + bean4 "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" bean2 "github.com/devtron-labs/devtron/pkg/workflow/dag/bean" error2 "github.com/devtron-labs/devtron/util/error" util2 "github.com/devtron-labs/devtron/util/event" @@ -74,7 +75,7 @@ type WorkflowDagExecutor interface { HandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt time.Time) (id int, err error) HandlePreStageSuccessEvent(triggerContext triggerBean.TriggerContext, cdStageCompleteEvent eventProcessorBean.CdStageCompleteEvent) error HandleDeploymentSuccessEvent(triggerContext triggerBean.TriggerContext, pipelineOverride *chartConfig.PipelineOverride) error - HandlePostStageSuccessEvent(triggerContext triggerBean.TriggerContext, cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error + HandlePostStageSuccessEvent(triggerContext triggerBean.TriggerContext, wfr *bean4.CdWorkflowRunnerDto, cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error HandleCdStageReTrigger(runner *pipelineConfig.CdWorkflowRunner) error HandleCiStepFailedEvent(ciPipelineId int, request *bean2.CiArtifactWebhookRequest) (err error) HandleExternalCiWebhook(externalCiId int, request *bean2.CiArtifactWebhookRequest, @@ -533,6 +534,19 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext t return err } if wfRunner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { + + pluginArtifacts := make(map[string][]string) + if cdStageCompleteEvent.PluginArtifacts != nil { + pluginArtifacts = cdStageCompleteEvent.PluginArtifacts.GetRegistryToUniqueContainerArtifactDataMapping() + util4.MergeMaps(pluginArtifacts, cdStageCompleteEvent.PluginRegistryArtifactDetails) + } + + err = impl.deactivateUnusedPaths(wfRunner.ImagePathReservationIds, pluginArtifacts) + if err != nil { + impl.logger.Errorw("error in deactiving unusedImagePaths", "err", err) + return err + } + pipeline, err := impl.pipelineRepository.FindById(cdStageCompleteEvent.CdPipelineId) if err != nil { return err @@ -548,7 +562,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext t impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", ciArtifact.Id) } } - PreCDArtifacts, err := impl.commonArtifactService.SavePluginArtifacts(ciArtifact, cdStageCompleteEvent.PluginRegistryArtifactDetails, pipeline.Id, repository.PRE_CD, cdStageCompleteEvent.TriggeredBy) + PreCDArtifacts, err := impl.commonArtifactService.SavePluginArtifacts(ciArtifact, pluginArtifacts, pipeline.Id, repository.PRE_CD, cdStageCompleteEvent.TriggeredBy) if err != nil { impl.logger.Errorw("error in saving plugin artifacts", "err", err) return err @@ -629,7 +643,7 @@ func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(triggerContext } else { // to trigger next pre/cd, if any // finding children cd by pipeline id - err = impl.HandlePostStageSuccessEvent(triggerContext, cdWorkflow.Id, pipelineOverride.PipelineId, 1, nil) + err = impl.HandlePostStageSuccessEvent(triggerContext, nil, cdWorkflow.Id, pipelineOverride.PipelineId, 1, nil) if err != nil { impl.logger.Errorw("error in triggering children cd after successful deployment event", "parentCdPipelineId", pipelineOverride.PipelineId) return err @@ -638,7 +652,7 @@ func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(triggerContext return nil } -func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(triggerContext triggerBean.TriggerContext, cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error { +func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(triggerContext triggerBean.TriggerContext, wfr *bean4.CdWorkflowRunnerDto, cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error { // finding children cd by pipeline id cdPipelinesMapping, err := impl.appWorkflowRepository.FindWFCDMappingByParentCDPipelineId(cdPipelineId) if err != nil { @@ -651,6 +665,13 @@ func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(triggerContext return err } if len(pluginRegistryImageDetails) > 0 { + if wfr != nil { + err = impl.deactivateUnusedPaths(wfr.ImagePathReservationIds, pluginRegistryImageDetails) + if err != nil { + impl.logger.Errorw("error in deactivation images", "err", err) + return err + } + } PostCDArtifacts, err := impl.commonArtifactService.SavePluginArtifacts(ciArtifact, pluginRegistryImageDetails, cdPipelineId, repository.POST_CD, triggeredBy) if err != nil { impl.logger.Errorw("error in saving plugin artifacts", "err", err) @@ -706,6 +727,13 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger impl.logger.Errorw("update wf failed for id ", "err", err) return 0, err } + + err = impl.deactivateUnusedPaths(savedWorkflow.ImagePathReservationIds, request.PluginRegistryArtifactDetails) + if err != nil { + impl.logger.Errorw("error in deactivation images", "err", err) + return 0, err + } + } pipeline, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) @@ -872,6 +900,37 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger return buildArtifact.Id, err } +func (impl *WorkflowDagExecutorImpl) deactivateUnusedPaths(reserveImagePathIds []int, pluginRegistryArtifactDetails map[string][]string) error { + // for copy container image plugin if images reserved are not equal to actual copird + reservedImagePaths, err := impl.customTagService.GetImagePathsByIds(reserveImagePathIds) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting imagePaths by ids", "ImagePathReservationIds", reserveImagePathIds, "err", err) + return err + } + + copiedImagesMapping := make(map[string]bool) + for _, savedImages := range pluginRegistryArtifactDetails { + for _, image := range savedImages { + copiedImagesMapping[image] = true + } + } + + unusedPaths := make([]string, 0, len(reservedImagePaths)) + for _, reservedImage := range reservedImagePaths { + if _, ok := copiedImagesMapping[reservedImage.ImagePath]; !ok { + unusedPaths = append(unusedPaths, reservedImage.ImagePath) + } + } + + err = impl.customTagService.DeactivateImagePathReservationByImagePath(unusedPaths) + if err != nil { + impl.logger.Errorw("error in deactivating unused image paths", "imagePathReservationIds", reserveImagePathIds, "err", err) + return err + } + + return nil +} + func (impl *WorkflowDagExecutorImpl) WriteCiSuccessEvent(request *bean2.CiArtifactWebhookRequest, pipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact) { event := impl.eventFactory.Build(util2.Success, &pipeline.Id, pipeline.AppId, nil, util2.CI) event.CiArtifactId = artifact.Id diff --git a/scripts/sql/286_copy_container_image_v2.down.sql b/scripts/sql/286_copy_container_image_v2.down.sql new file mode 100644 index 0000000000..b1b2291c11 --- /dev/null +++ b/scripts/sql/286_copy_container_image_v2.down.sql @@ -0,0 +1 @@ +delete from plugin_parent_metadata where identifier='copy-container-image'; \ No newline at end of file diff --git a/scripts/sql/286_copy_container_image_v2.up.sql b/scripts/sql/286_copy_container_image_v2.up.sql new file mode 100644 index 0000000000..34476367e8 --- /dev/null +++ b/scripts/sql/286_copy_container_image_v2.up.sql @@ -0,0 +1,25 @@ + + +INSERT INTO "plugin_parent_metadata" ("id", "name", "identifier", "description", "type", "icon", "deleted", "created_on", "created_by", "updated_on", "updated_by") +SELECT nextval('id_seq_plugin_parent_metadata'), 'Copy container image','copy-container-image', 'Copy container images from the source repository to a desired repository','PRESET','https://raw.githubusercontent.com/devtron-labs/devtron/main/assets/ic-plugin-copy-container-image.png','f', 'now()', 1, 'now()', 1 + WHERE NOT EXISTS ( + SELECT 1 + FROM plugin_parent_metadata + WHERE identifier='copy-container-image' + AND deleted = false +); + +-- update the plugin_metadata with the plugin_parent_metadata_id +UPDATE plugin_metadata +SET plugin_parent_metadata_id = ( + SELECT id + FROM plugin_parent_metadata + WHERE identifier='copy-container-image' + AND deleted = false +),plugin_version='1.0.0' +WHERE name='Copy container image' + AND ( + plugin_parent_metadata_id IS NULL + OR plugin_parent_metadata_id = 0 + ) + AND deleted = false; diff --git a/util/mapUtil.go b/util/mapUtil.go new file mode 100644 index 0000000000..07476ba8c6 --- /dev/null +++ b/util/mapUtil.go @@ -0,0 +1,13 @@ +package util + +func MergeMaps(map1, map2 map[string][]string) { + for key, values := range map2 { + if existingValues, found := map1[key]; found { + // Key exists in map1, append the values from map2 + map1[key] = append(existingValues, values...) + } else { + // Key does not exist in map1, add the new key-value pair + map1[key] = values + } + } +}