chore(deps): Migrate routing processor to routing connector for metrics collection

0.136.* version of otel collector does not support routing processor, the routing
processor is being moved to routing connector. This PR changes the routing config
for metrics.

Signed-off-by: Dhruv Shah <[email protected]>

Author: dhruv-shah-sumo

deploy/helm/sumologic/conf/logs/otelcol/config.yaml

@@ -428,38 +428,6 @@ processors:
       value: ""
 {{ end }}
 
-{{- if .Values.sumologic.logs.otelcol.routing.table }}
-  routing/containers:
-    default_exporters:
-{{ include "logs.otelcol.routing.defaultExporters" (dict "Values" .Values "Type" "containers") | nindent 6}}
-    table:
-{{- range $entry := .Values.sumologic.logs.otelcol.routing.table }}
-      - statement: {{ $entry.statement }}
-        exporters:
-          - {{ $entry.exporter }}
-{{- end }}
-{{- if .Values.sumologic.logs.otelcol.useDefaultExporters }}
-      - statement: route()
-        exporters:
-{{ include "logs.otelcol.routing.defaultExporters" (dict "Values" .Values "Type" "containers") | nindent 12}}
-{{- end }}
-
-  routing/systemd:
-    default_exporters:
-{{ include "logs.otelcol.routing.defaultExporters" (dict "Values" .Values "Type" "systemd") | nindent 6 }}
-    table:
-{{- range $entry := .Values.sumologic.logs.otelcol.routing.table }}
-      - statement: {{ $entry.statement }}
-        exporters:
-          - {{ $entry.exporter }}
-{{- end }}
-{{- if .Values.sumologic.logs.otelcol.useDefaultExporters }}
-      - statement: route()
-        exporters:
-{{ include "logs.otelcol.routing.defaultExporters" (dict "Values" .Values "Type" "systemd") | nindent 12 }}
-{{- end }}
-{{- end }}
-
 {{ if .Values.sumologic.logs.container.enabled }}
   source/containers:
     annotation_prefix: "pod_annotations_"
@@ -561,6 +529,39 @@ receivers:
         endpoint: {{ template "sumologic.otlp_http_endpoint" . }}
 {{ end }}
 
+{{- if .Values.sumologic.logs.otelcol.routing.table }}
+connectors:
+  routing/containers:
+    default_pipelines:
+      - logs/otlp/containers/routing/default
+    table:
+{{- range $entry := .Values.sumologic.logs.otelcol.routing.table }}
+      - statement: {{ $entry.statement }}
+        pipelines:
+{{- range $exp := $entry.exporters }}
+          - logs/otlp/containers/routing/{{ $exp }}
+{{- end }}
+{{- if $.Values.sumologic.logs.otelcol.useDefaultExporters }}
+          - logs/otlp/containers/routing/default
+{{- end}}
+{{- end}}
+
+  routing/systemd:
+    default_pipelines:
+      - logs/otlp/systemd/routing/default
+    table:
+{{- range $entry := .Values.sumologic.logs.otelcol.routing.table }}
+      - statement: {{ $entry.statement }}
+        pipelines:
+{{- range $exp := $entry.exporters }}
+          - logs/otlp/systemd/routing/{{ $exp }}
+{{- end }}
+{{- if $.Values.sumologic.logs.otelcol.useDefaultExporters }}
+          - logs/otlp/systemd/routing/default
+{{- end}}
+{{- end }}
+{{- end }}
+
 service:
   extensions:
     - health_check
@@ -573,7 +574,11 @@ service:
 {{ if .Values.sumologic.logs.container.enabled }}
     logs/otlp/containers:
       exporters:
+{{- if .Values.sumologic.logs.otelcol.routing.table }}
+        - routing/containers
+{{- else }}
 {{ include "logs.otelcol.exporters" (dict "Values" .Values "Type" "containers") | nindent 8}}
+{{- end }}
       processors:
         - memory_limiter
         - filter/include_containers
@@ -603,17 +608,18 @@ service:
         - transform/flatten
 {{- end }}
         - batch
-{{- if .Values.sumologic.logs.otelcol.routing.table }}
-        - routing/containers
-{{- end }}
       receivers:
         - otlp
 {{ end }}
 
 {{ if .Values.sumologic.logs.systemd.enabled }}
     logs/otlp/kubelet:
       exporters:
+{{- if .Values.sumologic.logs.otelcol.routing.table }}
+        - routing/systemd
+{{- else }}
 {{ include "logs.otelcol.exporters" (dict "Values" .Values "Type" "systemd") | nindent 8}}
+{{- end }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host
@@ -636,15 +642,16 @@ service:
 {{- end }}
 {{- end }}
         - batch
-{{- if .Values.sumologic.logs.otelcol.routing.table }}
-        - routing/systemd
-{{- end }}
       receivers:
         - otlp
 
     logs/otlp/systemd:
       exporters:
+{{- if .Values.sumologic.logs.otelcol.routing.table }}
+        - routing/systemd
+{{- else }}
 {{ include "logs.otelcol.exporters" (dict "Values" .Values "Type" "systemd") | nindent 8}}
+{{- end }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host
@@ -668,12 +675,45 @@ service:
 {{- end }}
 {{- end }}
         - batch
-{{- if .Values.sumologic.logs.otelcol.routing.table }}
-        - routing/systemd
-{{- end }}
       receivers:
         - otlp
 {{ end }}
+
+{{- if .Values.sumologic.logs.otelcol.routing.table }}
+{{- if .Values.sumologic.logs.container.enabled }}
+    logs/otlp/containers/routing/default:
+      receivers:
+        - routing/containers
+      exporters:
+{{ include "logs.otelcol.routing.defaultExporters" (dict "Values" .Values "Type" "containers") | nindent 8}}
+{{- range $entry := .Values.sumologic.logs.otelcol.routing.table }}
+{{- range $exp := $entry.exporters }}
+    logs/otlp/containers/routing/{{ $exp }}:
+      receivers:
+        - routing/containers
+      exporters:
+        - {{ $exp }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- if .Values.sumologic.logs.systemd.enabled }}
+    logs/otlp/systemd/routing/default:
+      receivers:
+        - routing/systemd
+      exporters:
+{{ include "logs.otelcol.routing.defaultExporters" (dict "Values" .Values "Type" "systemd") | nindent 8}}
+{{- range $entry := .Values.sumologic.logs.otelcol.routing.table }}
+{{- range $exp := $entry.exporters }}
+    logs/otlp/systemd/routing/{{ $exp }}:
+      receivers:
+        - routing/systemd
+      exporters:
+        - {{ $exp }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
 {{ end }}
   telemetry:
     logs:

deploy/helm/sumologic/conf/metrics/otelcol/config.yaml

@@ -1,3 +1,8 @@
+{{- if eq .Values.sumologic.metrics.sourceType "http" }}
+connectors:
+{{ tpl (.Files.Get "conf/metrics/otelcol/connectors.yaml") . | indent 2 }}
+{{- end}}
+
 exporters:
 {{ tpl (.Files.Get "conf/metrics/otelcol/exporters.yaml") . | indent 2 }}
 
@@ -53,6 +58,67 @@ service:
   pipelines:
     metrics:
 {{ tpl (.Files.Get "conf/metrics/otelcol/pipeline.yaml") . | indent 6 }}
+{{- if eq .Values.sumologic.metrics.sourceType "http" }}
+    metrics/sumologic/default:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/default
+{{- if eq .Values.debug.metrics.metadata.print true }}
+    metrics/debug:
+      receivers:
+        - routing
+      exporters:
+        - debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+    metrics/sumologic/sumologic-mock-default:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/sumologic-mock-default
+    metrics/sumologic/sumologic-mock-http:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/sumologic-mock-http
+{{- end }}
+    metrics/apiserver:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/apiserver
+    metrics/control_plane:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/control_plane
+    metrics/controller:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/controller
+    metrics/kubelet:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/kubelet
+    metrics/node:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/node
+    metrics/scheduler:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/scheduler
+    metrics/state:
+      receivers:
+        - routing
+      exporters:
+        - sumologic/state
+{{- end }}
   telemetry:
     logs:
       level: {{ .Values.metadata.metrics.logLevel }}

deploy/helm/sumologic/conf/metrics/otelcol/connectors.yaml

@@ -0,0 +1,82 @@
+routing:
+  default_pipelines:
+    - metrics/sumologic/default
+{{- if eq .Values.debug.metrics.metadata.print true }}
+    - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+    - metrics/sumologic/sumologic-mock-default
+{{- end }}
+  table:
+    - statement: route() where resource.attributes["job"] == "apiserver"
+      pipelines:
+        - metrics/apiserver
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "coredns"
+      pipelines:
+        - metrics/control_plane
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-etcd"
+      pipelines:
+        - metrics/control_plane
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-controller-manager"
+      pipelines:
+        - metrics/controller
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kubelet"
+      pipelines:
+        - metrics/kubelet
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "node-exporter"
+      pipelines:
+        - metrics/node
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-scheduler"
+      pipelines:
+        - metrics/scheduler
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-state-metrics"
+      pipelines:
+        - metrics/state
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}

deploy/helm/sumologic/conf/metrics/otelcol/pipeline.yaml

@@ -1,22 +1,14 @@
 exporters:
+{{- if ne .Values.sumologic.metrics.sourceType "http" }}
   - sumologic/default
 {{- if eq .Values.debug.metrics.metadata.print true }}
   - debug
 {{- end }}
 {{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
   - sumologic/sumologic-mock-default
 {{- end }}
-{{- if eq .Values.sumologic.metrics.sourceType "http" }}
-  - sumologic/apiserver
-  - sumologic/control_plane
-  - sumologic/controller
-  - sumologic/kubelet
-  - sumologic/node
-  - sumologic/scheduler
-  - sumologic/state
-{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
-  - sumologic/sumologic-mock-http
-{{- end }}
+{{- else }}
+  - routing
 {{- end }}
 processors:
   - memory_limiter
@@ -45,9 +37,6 @@ processors:
   - filter/app_metrics
 {{- end }}
   - batch
-{{- if eq .Values.sumologic.metrics.sourceType "http" }}
-  - routing
-{{- end }}
 receivers:
   - telegraf
   - otlp