Merge pull request #2 from oracle/setup-prometheus-exporter

Setup prometheus exporter

Author: vbhargav875

go.mod

@@ -12,11 +12,12 @@ require (
 	github.com/onsi/gomega v1.27.1
 	github.com/oracle/oci-go-sdk/v65 v65.34.0
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.14.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.26.1
 	k8s.io/apimachinery v0.26.1
 	k8s.io/client-go v0.26.1
-	k8s.io/klog/v2 v2.80.1
+	k8s.io/klog/v2 v2.90.0
 	sigs.k8s.io/controller-runtime v0.14.6
 )
 
@@ -47,7 +48,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/onsi/ginkgo/v2 v2.8.4 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect

go.sum

@@ -593,8 +593,8 @@ k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu7
 k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU=
 k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE=
 k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4=
-k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
-k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/klog/v2 v2.90.0 h1:VkTxIV/FjRXn1fgNNcKGM8cfmL1Z33ZjXRTVxKCoF5M=
+k8s.io/klog/v2 v2.90.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y=

helm/oci-native-ingress-controller/templates/deployment.yaml

@@ -61,6 +61,8 @@ spec:
           - --controller-class={{ .Values.controller_class }}
           - --compartment-id={{ .Values.compartment_id }}
           - --subnet-id={{ .Values.subnet_id }}
+          - --metrics-backend={{.Values.metrics.backend}}
+          - --metrics-port={{.Values.metrics.port}}
           - --v=4
           env:
             - name: OCI_RESOURCE_PRINCIPAL_VERSION
@@ -73,6 +75,8 @@ spec:
             - name: webhook-server
               containerPort: {{.Values.webhookBindPort}}
               protocol: TCP
+            - name: metrics-server
+              containerPort: 2223
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:

helm/oci-native-ingress-controller/values.yaml

@@ -106,3 +106,7 @@ objectSelector:
   #   - <value>
   matchLabels:
   #   key: value
+
+metrics:
+  backend: prometheus
+  port: 2223

main.go

@@ -11,11 +11,13 @@ package main
 import (
 	"context"
 	"flag"
+	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
+	"github.com/oracle/oci-native-ingress-controller/pkg/metric"
 	"github.com/oracle/oci-native-ingress-controller/pkg/server"
 	"github.com/oracle/oci-native-ingress-controller/pkg/types"
 	"k8s.io/klog/v2"
@@ -51,6 +53,8 @@ func main() {
 	flag.StringVar(&opts.ControllerClass, "controller-class", "oci.oraclecloud.com/native-ingress-controller", "the controller class name")
 	flag.StringVar(&opts.AuthType, "authType", "instance", "The auth type to be used - supported values : user, instance(Default).")
 	flag.StringVar(&opts.AuthSecretName, "auth-secret-name", "", "Secret name used for auth, cannot be empty if authType is user principal")
+	flag.StringVar(&opts.MetricsBackend, "metrics-backend", "prometheus", "Backend used for metrics")
+	flag.IntVar(&opts.MetricsPort, "metrics-port", 2223, "Metrics port for metrics backend")
 	flag.Parse()
 
 	if opts.LeaseLockName == "" {
@@ -157,9 +161,12 @@ func main() {
 	}
 
 	server.SetupWebhookServer(ingressInformer, serviceInformer, client, ctx)
+	mux := http.NewServeMux()
+	reg, err := server.SetupMetricsServer(opts.MetricsBackend, opts.MetricsPort, mux, ctx)
 
-	run := server.SetUpControllers(opts, ingressClassInformer, ingressInformer, client, serviceInformer, endpointInformer, podInformer, c)
+	run := server.SetUpControllers(opts, ingressClassInformer, ingressInformer, client, serviceInformer, endpointInformer, podInformer, c, reg)
 
+	metric.ServeMetrics(opts.MetricsPort, mux)
 	// we use the Lease lock type since edits to Leases are less common
 	// and fewer objects in the cluster watch "all Leases".
 	lock := &resourcelock.LeaseLock{

pkg/controllers/ingress/ingress.go

@@ -13,6 +13,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
 	"reflect"
 	"time"
 
@@ -23,6 +24,7 @@ import (
 
 	"github.com/oracle/oci-native-ingress-controller/pkg/certificate"
 	"github.com/oracle/oci-native-ingress-controller/pkg/loadbalancer"
+	"github.com/oracle/oci-native-ingress-controller/pkg/metric"
 	"github.com/oracle/oci-native-ingress-controller/pkg/state"
 	"github.com/oracle/oci-native-ingress-controller/pkg/util"
 
@@ -60,18 +62,14 @@ type Controller struct {
 
 	lbClient           *loadbalancer.LoadBalancerClient
 	certificatesClient *certificate.CertificatesClient
+	metricsCollector   *metric.IngressCollector
 }
 
 // NewController creates a new Controller.
-func NewController(
-	controllerClass string,
-	defaultCompartmentId string,
-	ingressClassInformer networkinginformers.IngressClassInformer,
-	ingressInformer networkinginformers.IngressInformer,
-	client kubernetes.Interface,
-	lbClient *loadbalancer.LoadBalancerClient,
-	certificatesClient *certificate.CertificatesClient,
-) *Controller {
+func NewController(controllerClass string, defaultCompartmentId string,
+	ingressClassInformer networkinginformers.IngressClassInformer, ingressInformer networkinginformers.IngressInformer,
+	client kubernetes.Interface, lbClient *loadbalancer.LoadBalancerClient, certificatesClient *certificate.CertificatesClient,
+	reg *prometheus.Registry) *Controller {
 
 	c := &Controller{
 		controllerClass:      controllerClass,
@@ -84,6 +82,7 @@ func NewController(
 		lbClient:           lbClient,
 		certificatesClient: certificatesClient,
 		queue:              workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(10*time.Second, 5*time.Minute)),
+		metricsCollector:   metric.NewIngressCollector(controllerClass, reg),
 	}
 
 	ingressInformer.Informer().AddEventHandler(
@@ -103,17 +102,18 @@ func (c *Controller) enqueueIngress(ingress *networkingv1.Ingress) {
 		utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", ingress, err))
 		return
 	}
-
 	c.queue.Add(key)
 }
 
 func (c *Controller) ingressAdd(obj interface{}) {
+	c.metricsCollector.IncrementIngressAddOperation()
 	ic := obj.(*networkingv1.Ingress)
 	klog.V(4).InfoS("Adding ingress", "ingress", klog.KObj(ic))
 	c.enqueueIngress(ic)
 }
 
 func (c *Controller) ingressUpdate(old, new interface{}) {
+	c.metricsCollector.IncrementIngressUpdateOperation()
 	oldIngress := old.(*networkingv1.Ingress)
 	newIngress := new.(*networkingv1.Ingress)
 
@@ -122,6 +122,7 @@ func (c *Controller) ingressUpdate(old, new interface{}) {
 }
 
 func (c *Controller) ingressDelete(obj interface{}) {
+	c.metricsCollector.IncrementIngressDeleteOperation()
 	ic, ok := obj.(*networkingv1.Ingress)
 	if !ok {
 		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
@@ -151,16 +152,23 @@ func (c *Controller) processNextItem() bool {
 	// parallel.
 	defer c.queue.Done(key)
 
+	// ingress_sync_time
+	startBuildTime := util.GetCurrentTimeInUnixMillis()
+
 	// Invoke the method containing the business logic
 	err := c.sync(key.(string))
 
+	endBuildTime := util.GetCurrentTimeInUnixMillis()
+	c.metricsCollector.AddIngressSyncTime(util.GetTimeDifferenceInSeconds(startBuildTime, endBuildTime))
+
 	// Handle the error if something went wrong during the execution of the business logic
 	c.handleErr(err, key)
 	return true
 }
 
 // sync is the business logic of the controller.
 func (c *Controller) sync(key string) error {
+	c.metricsCollector.IncrementSyncCount()
 	namespace, name, err := cache.SplitMetaNamespaceKey(key)
 	if err != nil {
 		klog.ErrorS(err, "Failed to split meta namespace cache key", "cacheKey", key)
@@ -285,7 +293,7 @@ func (c *Controller) ensureLoadBalancerIP(lbID string, ingress *networkingv1.Ing
 func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *networkingv1.IngressClass) error {
 
 	klog.Infof("Processing ingress %s/%s", ingressClass.Name, ingress.Name)
-	stateStore := state.NewStateStore(c.ingressClassLister, c.ingressLister, c.serviceLister)
+	stateStore := state.NewStateStore(c.ingressClassLister, c.ingressLister, c.serviceLister, c.metricsCollector)
 	ingressConfigError := stateStore.BuildState(ingressClass)
 
 	if ingressConfigError != nil {
@@ -316,6 +324,7 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *
 	backendSetsToCreate := desiredBackendSets.Difference(actualBackendSets)
 
 	for _, bsName := range backendSetsToCreate.List() {
+		startBuildTime := util.GetCurrentTimeInUnixMillis()
 		klog.V(2).InfoS("creating backend set for ingress", "ingress", klog.KObj(ingress), "backendSetName", bsName)
 		artifact, artifactType := stateStore.GetTLSConfigForBackendSet(bsName)
 		backendSetSslConfig, err := getSSLConfigForBackendSet(ingress.Namespace, artifactType, artifact, lb, bsName, c)
@@ -329,6 +338,8 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *
 		if err != nil {
 			return err
 		}
+		endBuildTime := util.GetCurrentTimeInUnixMillis()
+		c.metricsCollector.AddBackendCreationTime(util.GetTimeDifferenceInSeconds(startBuildTime, endBuildTime))
 	}
 
 	// Determine listeners... This is based off path ports.
@@ -380,7 +391,7 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *
 }
 
 func handleIngressDelete(c *Controller, ingressClass *networkingv1.IngressClass) error {
-	stateStore := state.NewStateStore(c.ingressClassLister, c.ingressLister, c.serviceLister)
+	stateStore := state.NewStateStore(c.ingressClassLister, c.ingressLister, c.serviceLister, c.metricsCollector)
 	ingressConfigError := stateStore.BuildState(ingressClass)
 
 	if ingressConfigError != nil {
@@ -577,6 +588,7 @@ func getSSLConfigForListener(namespace string, listener *ociloadbalancer.Listene
 }
 
 func syncListener(namespace string, stateStore *state.StateStore, lbId *string, listenerName string, c *Controller) error {
+	startTime := util.GetCurrentTimeInUnixMillis()
 	lb, etag, err := c.lbClient.GetLoadBalancer(context.TODO(), *lbId)
 	if err != nil {
 		return err
@@ -617,10 +629,14 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string,
 			return err
 		}
 	}
+	endTime := util.GetCurrentTimeInUnixMillis()
+	c.metricsCollector.AddIngressListenerSyncTime(util.GetTimeDifferenceInSeconds(startTime, endTime))
 	return nil
 }
 
 func syncBackendSet(ingress *networkingv1.Ingress, lbID string, backendSetName string, stateStore *state.StateStore, c *Controller) error {
+
+	startTime := util.GetCurrentTimeInUnixMillis()
 	lb, etag, err := c.lbClient.GetLoadBalancer(context.TODO(), lbID)
 	if err != nil {
 		return err
@@ -665,6 +681,8 @@ func syncBackendSet(ingress *networkingv1.Ingress, lbID string, backendSetName s
 		}
 	}
 
+	endTime := util.GetCurrentTimeInUnixMillis()
+	c.metricsCollector.AddIngressBackendSyncTime(util.GetTimeDifferenceInSeconds(startTime, endTime))
 	return nil
 }