chore(deps): Migrate routing processor to routing connector for metrics collection

0.136.* version of otel collector does not support routing processor, the routing
processor is being moved to routing connector. This PR changes the routing config
for metrics.

Signed-off-by: Dhruv Shah <[email protected]>

Author: dhruv-shah-sumo

deploy/helm/sumologic/conf/metrics/otelcol/config.yaml

@@ -1,3 +1,8 @@
+{{- if eq .Values.sumologic.metrics.sourceType "http" }}
+connectors:
+{{ tpl (.Files.Get "conf/metrics/otelcol/connectors.yaml") . | indent 2 }}
+{{- end}}
+
 exporters:
 {{ tpl (.Files.Get "conf/metrics/otelcol/exporters.yaml") . | indent 2 }}
 
@@ -53,6 +58,67 @@ service:
   pipelines:
     metrics:
 {{ tpl (.Files.Get "conf/metrics/otelcol/pipeline.yaml") . | indent 6 }}
+{{- if eq .Values.sumologic.metrics.sourceType "http" }}
+    metrics/sumologic/default:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/default
+{{- if eq .Values.debug.metrics.metadata.print true }}
+    metrics/debug:
+      receivers:
+        - routing/default
+      exporters:
+        - debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+    metrics/sumologic/sumologic-mock-default:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/sumologic-mock-default
+    metrics/sumologic/sumologic-mock-http:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/sumologic-mock-http
+{{- end }}
+    metrics/apiserver:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/apiserver
+    metrics/control_plane:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/control_plane
+    metrics/controller:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/controller
+    metrics/kubelet:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/kubelet
+    metrics/node:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/node
+    metrics/scheduler:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/scheduler
+    metrics/state:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/state
+{{- end }}
   telemetry:
     logs:
       level: {{ .Values.metadata.metrics.logLevel }}

deploy/helm/sumologic/conf/metrics/otelcol/connectors.yaml

@@ -0,0 +1,82 @@
+routing/default:
+  default_pipelines:
+    - metrics/sumologic/default
+{{- if eq .Values.debug.metrics.metadata.print true }}
+    - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+    - metrics/sumologic/sumologic-mock-default
+{{- end }}
+  table:
+    - statement: route() where resource.attributes["job"] == "apiserver"
+      pipelines:
+        - metrics/apiserver
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "coredns"
+      pipelines:
+        - metrics/control_plane
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-etcd"
+      pipelines:
+        - metrics/control_plane
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-controller-manager"
+      pipelines:
+        - metrics/controller
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kubelet"
+      pipelines:
+        - metrics/kubelet
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "node-exporter"
+      pipelines:
+        - metrics/node
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-scheduler"
+      pipelines:
+        - metrics/scheduler
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-state-metrics"
+      pipelines:
+        - metrics/state
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}

deploy/helm/sumologic/conf/metrics/otelcol/pipeline.yaml

@@ -1,22 +1,14 @@
 exporters:
+{{- if ne .Values.sumologic.metrics.sourceType "http" }}
   - sumologic/default
 {{- if eq .Values.debug.metrics.metadata.print true }}
   - debug
 {{- end }}
 {{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
   - sumologic/sumologic-mock-default
 {{- end }}
-{{- if eq .Values.sumologic.metrics.sourceType "http" }}
-  - sumologic/apiserver
-  - sumologic/control_plane
-  - sumologic/controller
-  - sumologic/kubelet
-  - sumologic/node
-  - sumologic/scheduler
-  - sumologic/state
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-  - sumologic/sumologic-mock-http
-{{- end }}
+{{- else }}
+  - routing/default
 {{- end }}
 processors:
   - memory_limiter
@@ -45,9 +37,6 @@ processors:
   - filter/app_metrics
 {{- end }}
   - batch
-{{- if eq .Values.sumologic.metrics.sourceType "http" }}
-  - routing
-{{- end }}
 receivers:
   - telegraf
   - otlp

deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml

@@ -165,98 +165,7 @@ resource/remove_k8s_pod_pod_name:
     - action: delete
       key: k8s.pod.pod_name
 
-{{- if eq .Values.sumologic.metrics.sourceType "http" }}
-routing:
-  default_exporters:
-    - sumologic/default
-{{- if eq .Values.debug.metrics.metadata.print true }}
-    - debug
-{{- end }}
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-    - sumologic/sumologic-mock-default
-{{- end }}
-  error_mode: ignore
-  table:
-    ## apiserver metrics
-    - exporters:
-        - sumologic/apiserver
-{{- if eq .Values.debug.metrics.metadata.print true }}
-        - debug
-{{- end }}
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-        - sumologic/sumologic-mock-http
-{{- end }}
-      statement: route() where resource.attributes["job"] == "apiserver"
-    ## control-plane metrics
-    - exporters:
-        - sumologic/control_plane
-{{- if eq .Values.debug.metrics.metadata.print true }}
-        - debug
-{{- end }}
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-        - sumologic/sumologic-mock-http
-{{- end }}
-      statement: route() where resource.attributes["job"] == "coredns"
-    - exporters:
-        - sumologic/control_plane
-{{- if eq .Values.debug.metrics.metadata.print true }}
-        - debug
-{{- end }}
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-        - sumologic/sumologic-mock-http
-{{- end }}
-      statement: route() where resource.attributes["job"] == "kube-etcd"
-    ## controller metrics
-    - exporters:
-        - sumologic/controller
-{{- if eq .Values.debug.metrics.metadata.print true }}
-        - debug
-{{- end }}
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-        - sumologic/sumologic-mock-http
-{{- end }}
-      statement: route() where resource.attributes["job"] == "kube-controller-manager"
-    ## kubelet metrics
-    - exporters:
-        - sumologic/kubelet
-{{- if eq .Values.debug.metrics.metadata.print true }}
-        - debug
-{{- end }}
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-        - sumologic/sumologic-mock-http
-{{- end }}
-      statement: route() where resource.attributes["job"] == "kubelet"
-    ## node metrics
-    - exporters:
-        - sumologic/node
-{{- if eq .Values.debug.metrics.metadata.print true }}
-        - debug
-{{- end }}
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-        - sumologic/sumologic-mock-http
-{{- end }}
-      statement: route() where resource.attributes["job"] == "node-exporter"
-    ## scheduler metrics
-    - exporters:
-        - sumologic/scheduler
-{{- if eq .Values.debug.metrics.metadata.print true }}
-        - debug
-{{- end }}
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-        - sumologic/sumologic-mock-http
-{{- end }}
-      statement: route() where resource.attributes["job"] == "kube-scheduler"
-    ## state metrics
-    - exporters:
-        - sumologic/state
-{{- if eq .Values.debug.metrics.metadata.print true }}
-        - debug
-{{- end }}
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-        - sumologic/sumologic-mock-http
-{{- end }}
-      statement: route() where resource.attributes["job"] == "kube-state-metrics"
-{{- end }}
+# routing removed to support routing connectors for the latest otel col version.
 
 ## Configuration for Source Processor
 ## Source processor adds Sumo Logic related metadata

deploy/helm/sumologic/values.yaml

@@ -462,11 +462,14 @@ sumologic:
         fallbackExporters: []
         table: []
         ## -
-        ##  ## exporter is name of the exporter
-        ##  exporter: sumologic/otlp
-        ##  ## statement is an OTTL condition which defines what data should be send to the exporter
-        ##  ## see routing processor documentation for more details:
-        ##  ## https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/aee4b75100530bce7edbf736fbcf76ac4f6ced6d/processor/routingprocessor/README.md#tech-preview-opentelemetry-transformation-language-statements-as-routing-conditions
+        ##  ## exporters is an array of the exporter
+        ##  exporters:
+        ##    - sumologic/otlp
+        ##  ## statement is an OTTL condition which defines what data should be sent to the exporters
+        ##  ## see routing connector documentation for more details:
+        ##  ## https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/aee4b75100530bce7edbf736fbcf76ac4f6ced6d/connector/routingconnector
+        ## routing connector expects you to define a pipeline, but here the user can just mention the exporter.
+        ## The helm deployment would convert exporters into correct pipelines and configure the connectors.
         ##  statement: ""
 
     ## Fields to be created at Sumo Logic to ensure logs are tagged with