Merge pull request #57 from BaizeAI/add-metrics

add metrics

Author: kebe7jun

README.md

@@ -1,8 +1,8 @@
 # snapshot-pod
-// TODO(user): Add simple overview of use/purpose
+Snapshot and export running Pod container images on Kubernetes nodes.
 
 ## Description
-// TODO(user): An in-depth paragraph about your project and overview of use
+This controller creates point-in-time snapshots of running Pods by committing container filesystem layers and optionally pushing the resulting image to a registry. It offers Prometheus metrics for observability of task states, failures, and controller health.
 
 ## Getting Started
 
@@ -66,6 +66,48 @@ make uninstall
 make undeploy
 ```
 
+## Metrics
+
+All metrics are exposed on the controller's Prometheus endpoint.
+
+- Snapshot tasks
+  - `snapshot_tasks_total{status, namespace, snapshot_name}`: Counter. Number of tasks by final/observed status. Use with rate/increase if needed.
+  - `snapshot_task_duration_seconds{status, namespace, snapshot_name}`: Histogram. Task duration seconds by status.
+  - `snapshot_task_retries_total{namespace, snapshot_name, task_name}`: Counter. Retry attempts count.
+  - `snapshot_active_tasks{namespace, snapshot_name}`: Gauge. Current active tasks per SnapshotPod.
+  - `snapshot_tasks_by_node{node_name, status}`: Gauge. Task counts per node and status.
+  - `snapshot_task_failed_total{namespace, snapshot_name, task_name}`: Counter. Increments when a task step errors or task exhausts max retries.
+
+- Controller
+  - `snapshot_controller_reconcile_total{controller, result}`: Counter. Reconcile outcomes (success/error/requeue).
+  - `snapshot_controller_reconcile_duration_seconds{controller}`: Histogram. Reconcile latency.
+  - `snapshot_controller_errors_total{controller, error_type}`: Counter. Controller error events.
+
+- Image push
+  - `snapshot_image_push_total{result, registry, namespace, snapshot_name}`: Counter. Push attempts.
+  - `snapshot_image_push_duration_seconds{registry, namespace, snapshot_name}`: Histogram. Push latency.
+
+### Alerting examples (PromQL)
+
+- New failures in last 5 minutes (per task):
+```promql
+increase(snapshot_task_failed_total[5m]) by (namespace, snapshot_name, task_name) > 0
+```
+
+- Any new failure in a namespace:
+```promql
+sum by (namespace) (increase(snapshot_task_failed_total[5m])) > 0
+```
+
+- Optional stability:
+```yaml
+for: 1m
+```
+
+Notes:
+- Always alert on a rate/increase over a window for counters; do not compare raw counters to 0.
+- Tune the lookback window (e.g., 5m/10m) to balance sensitivity vs stability.
+
 ## Project Distribution
 
 Following are the steps to build the installer and distribute this project to users.

cmd/main.go

@@ -43,6 +43,7 @@ import (
 
 	snapshotpodv1alpha1 "github.com/baizeai/kube-snapshot/api/v1alpha1"
 	"github.com/baizeai/kube-snapshot/internal/controller"
+	"github.com/baizeai/kube-snapshot/internal/metrics"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -66,8 +67,8 @@ func main() {
 	var enableHTTP2 bool
 	var systemWideDockerConfigPath string
 	var certDir string
-	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+
-		"Use the port :8080. If not set, it will be 0 in order to disable the metrics server")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to. "+
+		"Set to '0' to disable the metrics server.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -212,6 +213,11 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	// Start metrics collector with shared signal context
+	ctx := ctrl.SetupSignalHandler()
+	metricsCollector := metrics.NewMetricsCollector(mgr.GetClient())
+	go metricsCollector.Start(ctx)
+
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)
@@ -222,7 +228,7 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

internal/controller/snapshotpod_controller.go

@@ -37,6 +37,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	snapshotpodv1alpha1 "github.com/baizeai/kube-snapshot/api/v1alpha1"
+	"github.com/baizeai/kube-snapshot/internal/metrics"
 	"github.com/baizeai/kube-snapshot/pkg/apis/snapshotpod/v1alpha1"
 )
 
@@ -322,11 +323,15 @@ func renderNewImageName(originImage, format string, appendSnappedSuffix bool) (s
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
 func (r *SnapshotPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	startTime := time.Now()
 	logger := log.FromContext(ctx)
+
 	sp := snapshotpodv1alpha1.SnapshotPod{}
 	err := r.Client.Get(ctx, req.NamespacedName, &sp)
 	if err != nil {
 		logger.Error(err, "get instance error")
+		metrics.RecordControllerError("snapshotpod", "get_instance_error")
+		metrics.RecordControllerReconcile("snapshotpod", metrics.ControllerResultError, time.Since(startTime))
 		return ctrl.Result{}, err
 	}
 	if sp.Spec.TriggerRound <= 0 {
@@ -392,8 +397,11 @@ func (r *SnapshotPodReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if find || lo.ContainsBy(sp.Status.Conditions, func(item metav1.Condition) bool {
 		return item.Status != metav1.ConditionTrue
 	}) {
+		metrics.RecordControllerReconcile("snapshotpod", metrics.ControllerResultRequeue, time.Since(startTime))
 		return ctrl.Result{RequeueAfter: time.Second * 15}, nil
 	}
+
+	metrics.RecordControllerReconcile("snapshotpod", metrics.ControllerResultSuccess, time.Since(startTime))
 	return ctrl.Result{}, nil
 }
 

internal/controller/snapshotpodtask_controller.go

@@ -35,6 +35,8 @@ import (
 
 	snapshotpodv1alpha1 "github.com/baizeai/kube-snapshot/api/v1alpha1"
 	criruntime "github.com/baizeai/kube-snapshot/internal/controller/runtime"
+	"github.com/baizeai/kube-snapshot/internal/metrics"
+	"github.com/baizeai/kube-snapshot/pkg/apis/snapshotpod/v1alpha1"
 )
 
 const (
@@ -85,16 +87,33 @@ func (r *SnapshotPodTaskReconciler) getImageAuthWithSecret(ctx context.Context,
 	return &a, nil
 }
 
+// getSnapshotName extracts the snapshot name from the task's labels, falling back to the task name if not found
+func (r *SnapshotPodTaskReconciler) getSnapshotName(spt snapshotpodv1alpha1.SnapshotPodTask) string {
+	snapshotName := spt.Labels[v1alpha1.SnapshotNameLabel]
+	if snapshotName == "" {
+		return "unknown"
+	}
+	return snapshotName
+}
+
 func (r *SnapshotPodTaskReconciler) reconcilePushImage(ctx context.Context, spt *snapshotpodv1alpha1.SnapshotPodTask) error {
+	startTime := time.Now()
+
+	// Get snapshot name from label
+	snapshotName := r.getSnapshotName(*spt)
+
 	rtName, rt, _, err := r.getRuntimeAndContainerID(spt.Spec.ContainerID)
 	if err != nil {
+		metrics.RecordImagePush(metrics.ImagePushResultFailed, "unknown", spt.Namespace, snapshotName, time.Since(startTime))
 		return err
 	}
 	if !rt.ImageExists(ctx, spt.Spec.CommitImage) {
+		metrics.RecordImagePush(metrics.ImagePushResultFailed, "unknown", spt.Namespace, snapshotName, time.Since(startTime))
 		return fmt.Errorf("image %s not exists", spt.Spec.CommitImage)
 	}
 	auth, err := r.getImageAuthWithSecret(ctx, spt.Namespace, spt.Spec.RegistrySecretRef, spt.Spec.CommitImage)
 	if err != nil {
+		metrics.RecordImagePush(metrics.ImagePushResultFailed, "unknown", spt.Namespace, snapshotName, time.Since(startTime))
 		return err
 	}
 	if rtName == containerdRuntime {
@@ -103,7 +122,21 @@ func (r *SnapshotPodTaskReconciler) reconcilePushImage(ctx context.Context, spt
 		a, _ := r.getImageAuthWithSecret(ctx, spt.Namespace, spt.Spec.OriginRegistrySecretRef, spt.Spec.OriginImage)
 		_ = rt.Pull(ctx, spt.Spec.OriginImage, a, "--unpack=false")
 	}
-	return rt.Push(ctx, spt.Spec.CommitImage, auth)
+
+	// Extract image registry for metrics label
+	registry := "unknown"
+	if ref, err := docker.ParseReference("//" + spt.Spec.CommitImage); err == nil {
+		registry = reference.Domain(ref.DockerReference())
+	}
+
+	err = rt.Push(ctx, spt.Spec.CommitImage, auth)
+	if err != nil {
+		metrics.RecordImagePush(metrics.ImagePushResultFailed, registry, spt.Namespace, snapshotName, time.Since(startTime))
+		return err
+	}
+
+	metrics.RecordImagePush(metrics.ImagePushResultSuccess, registry, spt.Namespace, snapshotName, time.Since(startTime))
+	return nil
 }
 
 func (r *SnapshotPodTaskReconciler) reconcileCommit(ctx context.Context, spt *snapshotpodv1alpha1.SnapshotPodTask) error {
@@ -142,6 +175,7 @@ func (r *SnapshotPodTaskReconciler) reconcileAccept(ctx context.Context, spt *sn
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
 func (r *SnapshotPodTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	startTime := time.Now()
 	logger := log.FromContext(ctx)
 
 	spt := snapshotpodv1alpha1.SnapshotPodTask{}
@@ -150,6 +184,8 @@ func (r *SnapshotPodTaskReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		if errors.IsNotFound(err) {
 			return ctrl.Result{}, nil
 		}
+		metrics.RecordControllerError("snapshotpodtask", "get_instance_error")
+		metrics.RecordControllerReconcile("snapshotpodtask", metrics.ControllerResultError, time.Since(startTime))
 		return ctrl.Result{}, err
 	}
 	if spt.Spec.NodeName != r.NodeName {
@@ -208,6 +244,7 @@ func (r *SnapshotPodTaskReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 			spt.Status.Conditions[i].Status = metav1.ConditionFalse
 			spt.Status.Conditions[i].Message = err.Error()
 			lastError = err
+
 			// Update status immediately after error
 			if updateErr := r.Status().Update(ctx, &spt); updateErr != nil {
 				return ctrl.Result{}, updateErr
@@ -227,6 +264,9 @@ func (r *SnapshotPodTaskReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		}
 	}
 
+	// Get snapshot name from label for metrics
+	snapshotName := r.getSnapshotName(spt)
+
 	// Update phase based on conditions
 	switch {
 	case lo.EveryBy(spt.Status.Conditions, func(item metav1.Condition) bool {
@@ -243,6 +283,9 @@ func (r *SnapshotPodTaskReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	}):
 		spt.Status.RetryCount++
 		spt.Status.LastRetryTime = lo.ToPtr(metav1.Now())
+
+		// Record retry count when it's actually incremented
+		metrics.RecordSnapshotTaskRetry(spt.Namespace, snapshotName, spt.Name)
 		maxRetries := spt.Spec.MaxRetries
 		if maxRetries == 0 {
 			maxRetries = 3
@@ -252,6 +295,8 @@ func (r *SnapshotPodTaskReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 			logger.Info("task failed after max retries",
 				"retryCount", spt.Status.RetryCount,
 				"maxRetries", maxRetries)
+
+			metrics.RecordTaskFailed(spt.Namespace, snapshotName, spt.Name)
 		}
 	default:
 		spt.Status.Phase = snapshotpodv1alpha1.SnapshotPodTaskPhaseCreated
@@ -262,10 +307,34 @@ func (r *SnapshotPodTaskReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		return ctrl.Result{}, err
 	}
 
+	// Record metrics only when task reaches terminal state
+	// For running tasks, we should use gauge metrics instead of counter/histogram
+	if spt.Status.Phase == snapshotpodv1alpha1.SnapshotPodTaskPhaseCompleted ||
+		spt.Status.Phase == snapshotpodv1alpha1.SnapshotPodTaskPhaseFailed {
+
+		var status string
+		switch spt.Status.Phase {
+		case snapshotpodv1alpha1.SnapshotPodTaskPhaseCompleted:
+			status = metrics.SnapshotTaskStatusSuccess
+		case snapshotpodv1alpha1.SnapshotPodTaskPhaseFailed:
+			status = metrics.SnapshotTaskStatusFailed
+		}
+
+		// Calculate task duration
+		var duration time.Duration
+		if !spt.CreationTimestamp.Time.IsZero() {
+			duration = time.Since(spt.CreationTimestamp.Time)
+		}
+		metrics.RecordSnapshotTask(status, spt.Namespace, snapshotName, duration)
+	}
+
 	switch spt.Status.Phase {
 	case snapshotpodv1alpha1.SnapshotPodTaskPhaseFailed, snapshotpodv1alpha1.SnapshotPodTaskPhaseCompleted:
+		metrics.RecordControllerReconcile("snapshotpodtask", metrics.ControllerResultSuccess, time.Since(startTime))
 		return ctrl.Result{}, nil
 	}
+
+	metrics.RecordControllerReconcile("snapshotpodtask", metrics.ControllerResultRequeue, time.Since(startTime))
 	retryDelay := time.Second * 30
 	if spt.Spec.RetryDelaySeconds > 0 {
 		retryDelay = time.Second * time.Duration(spt.Spec.RetryDelaySeconds)