fix: handle config scraper deletion/un-deletion and cleanup (#1749)

* fix: handle config scraper deletion/un-deletion and cleanup

* chore: unset config_scraper deleted_at

* chore: remove debug

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

---------

Co-authored-by: Moshe Immerman <[email protected]>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

Author: 3 people

api/v1/types.go

@@ -232,4 +232,6 @@ var (
 	DeletedReasonFromDeleteField ConfigDeleteReason = "FROM_DELETE_FIELD"
 
 	DeleteReasonEvent ConfigDeleteReason = "FROM_EVENT"
+
+	DeleteReasonScraperDeleted ConfigDeleteReason = "SCRAPER_DELETED"
 )

db/config_scraper.go

@@ -60,11 +60,13 @@ func DeleteScrapeConfig(ctx context.Context, id string) error {
 	// Soft delete remaining config_items
 	if err := ctx.DB().Exec(fmt.Sprintf(`
         UPDATE config_items
-        SET deleted_at = NOW()
+        SET
+			deleted_at = NOW(),
+			delete_reason = ?
         WHERE id NOT IN (%s)
 			AND scraper_id = ?
 			AND deleted_at IS NULL
-    `, selectQuery), id).Error; err != nil {
+    `, selectQuery), v1.DeleteReasonScraperDeleted, id).Error; err != nil {
 		return err
 	}
 	return nil
@@ -109,8 +111,19 @@ func PersistScrapeConfigFromCRD(ctx context.Context, scrapeConfig *v1.ScrapeConf
 		Spec:      spec,
 		Source:    models.SourceCRD,
 	}
-	tx := ctx.DB().Save(&configScraper)
-	return configScraper, changed, tx.Error
+	if err := ctx.DB().Save(&configScraper).Error; err != nil {
+		return models.ConfigScraper{}, changed, err
+	}
+
+	if existing.DeletedAt != nil {
+		if err := ctx.DB().Model(&models.ConfigScraper{}).
+			Where("id = ?", configScraper.ID).
+			Update("deleted_at", gorm.Expr("NULL")).Error; err != nil {
+			return models.ConfigScraper{}, changed, fmt.Errorf("error reseting deleted_at for scraper[%s]: %w", configScraper.ID, err)
+		}
+	}
+
+	return configScraper, changed, nil
 }
 
 func GetScrapeConfigsOfAgent(ctx context.Context, agentID uuid.UUID) ([]models.ConfigScraper, error) {

jobs/cleanup.go

@@ -26,6 +26,7 @@ var (
 	ConfigAnalysisRetentionDays int
 	ConfigChangeRetentionDays   int
 	ConfigItemRetentionDays     int
+	ConfigScraperRetentionDays  = 30
 )
 
 var cleanupJobs = []*job.Job{
@@ -177,12 +178,19 @@ var CleanupConfigScrapers = &job.Job{
 
 		for _, id := range deletedIDs {
 			if err := cdb.DeleteScrapeConfig(ctx.Context, id); err != nil {
-				ctx.History.AddErrorf("error deleting scrape config[%s]: %v", id, err)
+				ctx.History.AddErrorf("error soft deleting scrape config[%s]: %v", id, err)
 			} else {
 				ctx.History.SuccessCount += 1
 			}
 		}
 
+		// Hard delete old config scrapers
+		retention := ctx.Properties().Duration("config_scraper.retention.period", (time.Hour * 24 * time.Duration(ConfigScraperRetentionDays)))
+		days := int64(retention.Hours() / 24)
+		if err := ctx.DB().Exec(`DELETE FROM config_scrapers WHERE (NOW() - deleted_at ) > INTERVAL '1 day' * ?`, days).Error; err != nil {
+			ctx.History.AddErrorf("error hard deleting config_scrapers: %v", err)
+		}
+
 		return nil
 	},
 }