diff --git a/backend/plugins/argocd/impl/impl.go b/backend/plugins/argocd/impl/impl.go index c55958ea188..77f2696a4b8 100644 --- a/backend/plugins/argocd/impl/impl.go +++ b/backend/plugins/argocd/impl/impl.go @@ -85,6 +85,7 @@ func (p ArgoCD) GetTablesInfo() []dal.Tabler { &models.ArgocdConnection{}, &models.ArgocdApplication{}, &models.ArgocdSyncOperation{}, + &models.ArgocdRevisionImage{}, &models.ArgocdScopeConfig{}, } } diff --git a/backend/plugins/argocd/models/application.go b/backend/plugins/argocd/models/application.go index 465cf0266fe..fd15a098bb1 100644 --- a/backend/plugins/argocd/models/application.go +++ b/backend/plugins/argocd/models/application.go @@ -39,6 +39,7 @@ type ArgocdApplication struct { SyncStatus string `gorm:"type:varchar(100)" json:"syncStatus" mapstructure:"syncStatus"` // Synced, OutOfSync, Unknown HealthStatus string `gorm:"type:varchar(100)" json:"healthStatus" mapstructure:"healthStatus"` // Healthy, Progressing, Degraded, Suspended, Missing, Unknown CreatedDate *time.Time `json:"createdDate,omitempty" mapstructure:"createdDate,omitempty"` + SummaryImages []string `gorm:"type:json;serializer:json" json:"summaryImages" mapstructure:"summaryImages"` common.NoPKModel } diff --git a/backend/plugins/argocd/models/migrationscripts/20251102_add_image_support.go b/backend/plugins/argocd/models/migrationscripts/20251102_add_image_support.go new file mode 100644 index 00000000000..7392d320c64 --- /dev/null +++ b/backend/plugins/argocd/models/migrationscripts/20251102_add_image_support.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/argocd/models/migrationscripts/archived" +) + +var _ plugin.MigrationScript = (*addImageSupportArtifacts)(nil) + +type addImageSupportArtifacts struct{} + +func (m *addImageSupportArtifacts) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.ArgocdApplication{}, + &archived.ArgocdSyncOperation{}, + &archived.ArgocdRevisionImage{}, + ) +} + +func (*addImageSupportArtifacts) Version() uint64 { + return 20251102160000 +} + +func (*addImageSupportArtifacts) Name() string { + return "argocd add image support artifacts" +} diff --git a/backend/plugins/argocd/models/migrationscripts/archived/models.go b/backend/plugins/argocd/models/migrationscripts/archived/models.go index ebaaa099cf5..2526c15cb97 100644 --- a/backend/plugins/argocd/models/migrationscripts/archived/models.go +++ b/backend/plugins/argocd/models/migrationscripts/archived/models.go @@ -47,6 +47,7 @@ type ArgocdApplication struct { SyncStatus string `gorm:"type:varchar(100)"` HealthStatus string `gorm:"type:varchar(100)"` CreatedDate *time.Time + SummaryImages []string `gorm:"type:json;serializer:json"` ScopeConfigId uint64 archived.NoPKModel } @@ -69,6 +70,7 @@ type ArgocdSyncOperation struct { SyncStatus string `gorm:"type:varchar(100)"` HealthStatus string `gorm:"type:varchar(100)"` ResourcesCount int + ContainerImages []string `gorm:"type:json;serializer:json"` archived.NoPKModel } @@ -76,6 +78,18 @@ func (ArgocdSyncOperation) TableName() string { return "_tool_argocd_sync_operations" } +type ArgocdRevisionImage struct { + ConnectionId uint64 `gorm:"primaryKey"` + ApplicationName string `gorm:"primaryKey;type:varchar(255)"` + Revision string `gorm:"primaryKey;type:varchar(255)"` + Images []string `gorm:"type:json;serializer:json"` + archived.NoPKModel +} + +func (ArgocdRevisionImage) TableName() string { + return "_tool_argocd_revision_images" +} + type ArgocdScopeConfig struct { archived.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` ConnectionId uint64 `gorm:"index"` diff --git a/backend/plugins/argocd/models/migrationscripts/register.go b/backend/plugins/argocd/models/migrationscripts/register.go index b223622cca3..2c70d811548 100644 --- a/backend/plugins/argocd/models/migrationscripts/register.go +++ b/backend/plugins/argocd/models/migrationscripts/register.go @@ -24,5 +24,6 @@ import ( func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables), + new(addImageSupportArtifacts), } } diff --git a/backend/plugins/argocd/models/revision_image.go b/backend/plugins/argocd/models/revision_image.go new file mode 100644 index 00000000000..7b1080f3103 --- /dev/null +++ b/backend/plugins/argocd/models/revision_image.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "github.com/apache/incubator-devlake/core/models/common" + +// ArgocdRevisionImage captures the container images observed for a given +// Argo CD application revision. It enables historical lookups so that +// previously processed sync operations retain the images that were active +// when they first ran, even after subsequent deployments update the +// application summary images. +type ArgocdRevisionImage struct { + ConnectionId uint64 `gorm:"primaryKey"` + ApplicationName string `gorm:"primaryKey;type:varchar(255)"` + Revision string `gorm:"primaryKey;type:varchar(255)"` + Images []string `gorm:"type:json;serializer:json"` + common.NoPKModel +} + +func (ArgocdRevisionImage) TableName() string { + return "_tool_argocd_revision_images" +} diff --git a/backend/plugins/argocd/models/sync_operation.go b/backend/plugins/argocd/models/sync_operation.go index 3f2f2c8339f..77cc095447d 100644 --- a/backend/plugins/argocd/models/sync_operation.go +++ b/backend/plugins/argocd/models/sync_operation.go @@ -37,6 +37,7 @@ type ArgocdSyncOperation struct { SyncStatus string `gorm:"type:varchar(100)"` // Synced, OutOfSync HealthStatus string `gorm:"type:varchar(100)"` // Healthy, Degraded, etc. ResourcesCount int + ContainerImages []string `gorm:"type:json;serializer:json"` common.NoPKModel } diff --git a/backend/plugins/argocd/tasks/application_extractor.go b/backend/plugins/argocd/tasks/application_extractor.go index 54a01392d40..3a0568609a5 100644 --- a/backend/plugins/argocd/tasks/application_extractor.go +++ b/backend/plugins/argocd/tasks/application_extractor.go @@ -63,6 +63,9 @@ type ArgocdApiApplication struct { Health struct { Status string `json:"status"` // Healthy, Progressing, Degraded, etc. } `json:"health"` + Summary struct { + Images []string `json:"images"` + } `json:"summary"` } `json:"status"` } @@ -96,6 +99,7 @@ func ExtractApplications(taskCtx plugin.SubTaskContext) errors.Error { DestNamespace: apiApp.Spec.Destination.Namespace, SyncStatus: apiApp.Status.Sync.Status, HealthStatus: apiApp.Status.Health.Status, + SummaryImages: apiApp.Status.Summary.Images, CreatedDate: &apiApp.Metadata.CreationTimestamp, } application.ConnectionId = data.Options.ConnectionId diff --git a/backend/plugins/argocd/tasks/sync_operation_extractor.go b/backend/plugins/argocd/tasks/sync_operation_extractor.go index b731149479c..5de738a38cc 100644 --- a/backend/plugins/argocd/tasks/sync_operation_extractor.go +++ b/backend/plugins/argocd/tasks/sync_operation_extractor.go @@ -19,8 +19,11 @@ package tasks import ( "encoding/json" + "sort" + "strings" "time" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -52,6 +55,8 @@ type ArgocdApiSyncOperation struct { Username string `json:"username"` Automated bool `json:"automated"` } `json:"initiatedBy"` + Metadata ArgocdApiSyncOperationMetadata `json:"metadata"` + Operation ArgocdApiSyncOperationDetails `json:"operation"` // For operationState (current operation) Phase string `json:"phase"` // Succeeded, Failed, Error, Running, Terminating @@ -65,18 +70,74 @@ type ArgocdApiSyncOperation struct { } type ArgocdApiSyncResourceItem struct { - Group string `json:"group"` - Version string `json:"version"` - Kind string `json:"kind"` - Namespace string `json:"namespace"` - Name string `json:"name"` - Status string `json:"status"` - Message string `json:"message"` + Group string `json:"group"` + Version string `json:"version"` + Kind string `json:"kind"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Status string `json:"status"` + Message string `json:"message"` + Images []string `json:"images"` +} + +type ArgocdApiSyncOperationMetadata struct { + Images []string `json:"images"` + Resources []ArgocdApiSyncOperationMetadataResource `json:"resources"` +} + +type ArgocdApiSyncOperationMetadataResource struct { + Images []string `json:"images"` +} + +type ArgocdApiSyncOperationDetails struct { + Metadata ArgocdApiSyncOperationMetadata `json:"metadata"` + Sync ArgocdApiSyncOperationSync `json:"sync"` +} + +type ArgocdApiSyncOperationSync struct { + Resources []ArgocdApiSyncResourceItem `json:"resources"` } func ExtractSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*ArgocdTaskData) + var summaryImages []string + application := &models.ArgocdApplication{} + db := taskCtx.GetDal() + if err := db.First( + application, + dal.Where("connection_id = ? AND name = ?", data.Options.ConnectionId, data.Options.ApplicationName), + ); err != nil { + if !db.IsErrorNotFound(err) { + return errors.Default.Wrap(err, "error loading argocd application for summary images") + } + } else { + summaryImages = application.SummaryImages + } + summaryImages = normalizeImages(summaryImages) + + revisionImageCache := make(map[string][]string) + revisionRecords := make([]models.ArgocdRevisionImage, 0) + err := db.All( + &revisionRecords, + dal.Where("connection_id = ? AND application_name = ?", data.Options.ConnectionId, data.Options.ApplicationName), + ) + if err != nil && !db.IsErrorNotFound(err) { + return errors.Default.Wrap(err, "error loading argocd revision images") + } + for _, record := range revisionRecords { + if record.Revision == "" { + continue + } + normalized := normalizeImages(record.Images) + if len(normalized) == 0 { + continue + } + revisionImageCache[record.Revision] = normalized + } + + revisionDirty := make(map[string][]string) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -160,7 +221,47 @@ func ExtractSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { syncOp.Kind = extractPrimaryDeploymentKind(apiOp.SyncResult.Resources) - return []interface{}{syncOp}, nil + payloadImages := collectContainerImages(&apiOp) + images := copyStringSlice(payloadImages) + + if len(images) > 0 { + if syncOp.Revision != "" { + cached := revisionImageCache[syncOp.Revision] + if !stringSlicesEqual(cached, images) { + revisionImageCache[syncOp.Revision] = copyStringSlice(images) + revisionDirty[syncOp.Revision] = copyStringSlice(images) + } + } + } else if syncOp.Revision != "" { + if cached := revisionImageCache[syncOp.Revision]; len(cached) > 0 { + images = copyStringSlice(cached) + } else if len(summaryImages) > 0 { + // Fallback: use application summary images and cache for this revision. + images = copyStringSlice(summaryImages) + revisionImageCache[syncOp.Revision] = copyStringSlice(images) + revisionDirty[syncOp.Revision] = copyStringSlice(images) + } + } + + if len(images) > 0 { + syncOp.ContainerImages = images + } + + results := []interface{}{syncOp} + if syncOp.Revision != "" { + if dirtyImages, ok := revisionDirty[syncOp.Revision]; ok && len(dirtyImages) > 0 { + revision := &models.ArgocdRevisionImage{ + ConnectionId: syncOp.ConnectionId, + ApplicationName: syncOp.ApplicationName, + Revision: syncOp.Revision, + Images: copyStringSlice(dirtyImages), + } + results = append(results, revision) + delete(revisionDirty, syncOp.Revision) + } + } + + return results, nil }, }) @@ -201,3 +302,80 @@ func extractPrimaryDeploymentKind(resources []ArgocdApiSyncResourceItem) string return "" } + +func collectContainerImages(apiOp *ArgocdApiSyncOperation) []string { + if apiOp == nil { + return nil + } + + var collected []string + appendAll := func(images []string) { + if len(images) == 0 { + return + } + collected = append(collected, images...) + } + appendMetadataResources := func(resources []ArgocdApiSyncOperationMetadataResource) { + for _, r := range resources { + appendAll(r.Images) + } + } + appendResourceItems := func(resources []ArgocdApiSyncResourceItem) { + for _, r := range resources { + appendAll(r.Images) + } + } + + appendAll(apiOp.Metadata.Images) + appendMetadataResources(apiOp.Metadata.Resources) + appendAll(apiOp.Operation.Metadata.Images) + appendMetadataResources(apiOp.Operation.Metadata.Resources) + appendResourceItems(apiOp.Operation.Sync.Resources) + appendResourceItems(apiOp.SyncResult.Resources) + + return normalizeImages(collected) +} + +func normalizeImages(images []string) []string { + if len(images) == 0 { + return nil + } + uniq := make(map[string]struct{}, len(images)) + for _, image := range images { + trimmed := strings.TrimSpace(image) + if trimmed == "" { + continue + } + uniq[trimmed] = struct{}{} + } + if len(uniq) == 0 { + return nil + } + normalized := make([]string, 0, len(uniq)) + for image := range uniq { + normalized = append(normalized, image) + } + sort.Strings(normalized) + return normalized +} + +func copyStringSlice(values []string) []string { + if len(values) == 0 { + return nil + } + dup := make([]string, len(values)) + copy(dup, values) + return dup +} + +func stringSlicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/backend/plugins/argocd/tasks/sync_operation_extractor_test.go b/backend/plugins/argocd/tasks/sync_operation_extractor_test.go new file mode 100644 index 00000000000..9d71cec41af --- /dev/null +++ b/backend/plugins/argocd/tasks/sync_operation_extractor_test.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCollectContainerImages_ReturnsSortedUniqueImages(t *testing.T) { + op := &ArgocdApiSyncOperation{} + op.Metadata.Images = []string{"registry.example.com/system:ops", " registry.example.com/sidecar:def456 "} + op.Metadata.Resources = []ArgocdApiSyncOperationMetadataResource{ + {Images: []string{"registry.example.com/app:abc123", "", "registry.example.com/api:789xyz"}}, + } + op.Operation.Metadata.Images = []string{"registry.example.com/worker:alpha"} + op.Operation.Metadata.Resources = []ArgocdApiSyncOperationMetadataResource{ + {Images: []string{"registry.example.com/rollout:blue"}}, + } + op.Operation.Sync.Resources = []ArgocdApiSyncResourceItem{ + {Images: []string{"registry.example.com/canary:latest"}}, + } + op.SyncResult.Resources = []ArgocdApiSyncResourceItem{ + {Images: []string{"registry.example.com/worker:alpha", "registry.example.com/api:789xyz"}}, + } + + images := collectContainerImages(op) + + expected := []string{ + "registry.example.com/api:789xyz", + "registry.example.com/app:abc123", + "registry.example.com/canary:latest", + "registry.example.com/rollout:blue", + "registry.example.com/sidecar:def456", + "registry.example.com/system:ops", + "registry.example.com/worker:alpha", + } + assert.Equal(t, expected, images) +} + +func TestCollectContainerImages_EmptyInputReturnsNil(t *testing.T) { + op := &ArgocdApiSyncOperation{} + + assert.Nil(t, collectContainerImages(op)) + assert.Nil(t, collectContainerImages(nil)) +} + +func TestNormalizeImages(t *testing.T) { + input := []string{" registry.example.com/app:1.0 ", "registry.example.com/app:1.0", "registry.example.com/api:2.0", ""} + expected := []string{"registry.example.com/api:2.0", "registry.example.com/app:1.0"} + assert.Equal(t, expected, normalizeImages(input)) +} + +// Fallback: no images in payload → expect none here (other fallbacks tested elsewhere) +func TestCollectContainerImages_FallbackRevisionAndSummary(t *testing.T) { + revision := "abcdef1234567890" + apiPayload := ArgocdApiSyncOperation{Revision: revision} + assert.Nil(t, collectContainerImages(&apiPayload)) + + // normalizeImages: dedupe + sort + assert.Equal(t, []string{"a", "b"}, normalizeImages([]string{"b", "a", "b"})) +} diff --git a/grafana/dashboards/ArgoCD.json b/grafana/dashboards/ArgoCD.json index 1086d47a607..262a6b5e05a 100644 --- a/grafana/dashboards/ArgoCD.json +++ b/grafana/dashboards/ArgoCD.json @@ -827,6 +827,136 @@ ], "title": "4.1 Recent Deployments", "type": "table" + }, + { + "collapsed": false, + "id": 17, + "panels": [], + "title": "5. Images", + "type": "row" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 18, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "deployment_created" + } + ] + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n d.created_date AS deployment_created,\n d.name AS deployment_name,\n ri.images AS images,\n c.commit_sha AS revision,\n d.environment,\n d.result\nFROM cicd_deployments d\nLEFT JOIN cicd_deployment_commits c ON c.cicd_deployment_id = d.id\nLEFT JOIN _tool_argocd_revision_images ri ON ri.revision = c.commit_sha\nWHERE $__timeFilter(d.created_date)\n AND ( '${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR FIND_IN_SET(d.cicd_scope_id, '${application_id:raw}') > 0 )\nORDER BY d.created_date DESC\nLIMIT 50", + "refId": "A" + } + ], + "title": "5.1 Recent Deployment Images", + "type": "table" + }, + { + "datasource": "mysql", + "description": "Approximation: counts deployments per unique images array (revision-level). For per-image counts, consider exploding arrays during ingestion.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 19, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH dep AS (\n SELECT d.id, c.commit_sha, d.cicd_scope_id\n FROM cicd_deployments d\n LEFT JOIN cicd_deployment_commits c ON c.cicd_deployment_id = d.id\n WHERE $__timeFilter(d.created_date)\n AND ( '${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR FIND_IN_SET(d.cicd_scope_id, '${application_id:raw}') > 0 )\n), imgs AS (\n SELECT dep.id deployment_id, ri.images\n FROM dep\n LEFT JOIN _tool_argocd_revision_images ri ON ri.revision = dep.commit_sha\n WHERE ri.images IS NOT NULL\n)\nSELECT images AS image_array, COUNT(DISTINCT deployment_id) deployment_count\nFROM imgs\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 20", + "refId": "A" + } + ], + "title": "5.2 Top Image Arrays (Approx)", + "type": "table" } ], "refresh": "",