add eval support in traceloop

verify commit

Author: adityamehra

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/cronjob-tl.yaml

@@ -0,0 +1,150 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: travel-planner-tl
+  namespace: o11y-4-ai-admehra
+  labels:
+    app: travel-planner-tl
+    component: telemetry
+  annotations:
+    description: "Multi-agent travel planner with Traceloop translator and GenAI evaluations (Deepeval telemetry disabled)"
+    git-commit: "30c512d"
+spec:
+  # Run every 30 minutes from 8 AM to 5 PM PST on weekdays (Monday-Friday)
+  schedule: "*/30 8-17 * * 1-5"
+  timeZone: "America/Los_Angeles"
+  suspend: false
+  
+  # Keep last 3 successful and 1 failed job for debugging
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 1
+  
+  jobTemplate:
+    metadata:
+      labels:
+        app: travel-planner-tl
+        component: telemetry
+    spec:
+      template:
+        metadata:
+          labels:
+            app: travel-planner-tl
+            component: telemetry
+        spec:
+          restartPolicy: OnFailure
+          
+          containers:
+            - name: travel-planner-traceloop
+              # Multi-platform image (amd64, arm64) with git commit hash tag
+              image: admehra621/travel-planner-tl:30c512d
+              imagePullPolicy: Always
+              
+              env:
+                # === GenAI Semantic Conventions (REQUIRED) ===
+                - name: OTEL_SEMCONV_STABILITY_OPT_IN
+                  value: "gen_ai_latest_experimental"
+                
+                # === OpenTelemetry Resource Attributes ===
+                - name: OTEL_RESOURCE_ATTRIBUTES
+                  value: "deployment.environment=o11y-inframon-ai,git.commit.id=30c512d"
+                
+                # === Service name for telemetry ===
+                - name: OTEL_SERVICE_NAME
+                  value: "travel-planner-tl"
+                
+                # === OpenAI Configuration ===
+                - name: OPENAI_API_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: openai-credentials
+                      key: api-key
+                
+                - name: OPENAI_MODEL
+                  value: "gpt-4o-mini"
+                
+                # === Deepeval Telemetry Opt-Out ===
+                - name: DEEPEVAL_TELEMETRY_OPT_OUT
+                  value: "1"
+                
+                # === GenAI Content Capture ===
+                - name: OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT
+                  value: "true"
+                
+                - name: OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT_MODE
+                  value: "SPAN_AND_EVENT"
+                
+                # === GenAI Emitters Configuration ===
+                - name: OTEL_INSTRUMENTATION_GENAI_EMITTERS
+                  value: "span_metric_event,splunk"
+                
+                - name: OTEL_INSTRUMENTATION_GENAI_EMITTERS_EVALUATION
+                  value: "replace-category:SplunkEvaluationResults"
+                
+                # === Evaluation Settings ===
+                # All 5 default evaluations enabled (bias, toxicity, relevance, hallucination, sentiment)
+                - name: OTEL_INSTRUMENTATION_GENAI_EVALS_RESULTS_AGGREGATION
+                  value: "true"
+                
+                # === GenAI Debug Flags (disabled for production) ===
+                # - name: OTEL_GENAI_EVAL_DEBUG_SKIPS
+                #   value: "true"
+                # - name: OTEL_GENAI_EVAL_DEBUG_EACH
+                #   value: "true"
+                # - name: OTEL_INSTRUMENTATION_GENAI_DEBUG
+                #   value: "true"
+                
+                # === OpenTelemetry Logs Exporter ===
+                - name: OTEL_LOGS_EXPORTER
+                  value: "otlp"
+                
+                # === Get the host IP for Splunk OTEL agent ===
+                - name: SPLUNK_OTEL_AGENT
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: status.hostIP
+                
+                # === OpenTelemetry OTLP endpoint using Splunk agent ===
+                - name: OTEL_EXPORTER_OTLP_ENDPOINT
+                  value: "http://$(SPLUNK_OTEL_AGENT):4317"
+                
+                # === OTLP Protocol (grpc) ===
+                - name: OTEL_EXPORTER_OTLP_PROTOCOL
+                  value: "grpc"
+                
+                # === Exclude health check URLs ===
+                - name: OTEL_PYTHON_EXCLUDED_URLS
+                  value: "^(https?://)?[^/]+(/)?$"
+                
+                # === Enable Python logging auto instrumentation ===
+                - name: OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED
+                  value: "true"
+                
+                # === Enable log correlation ===
+                - name: OTEL_PYTHON_LOG_CORRELATION
+                  value: "true"
+                
+                # === Enable LangChain content capture ===
+                - name: OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT
+                  value: "true"
+                
+                # === Enable Splunk profiler ===
+                - name: SPLUNK_PROFILER_ENABLED
+                  value: "true"
+                
+                # === Unbuffered Python output ===
+                - name: PYTHONUNBUFFERED
+                  value: "1"
+                
+                # === GenAI evaluation sampling rate ===
+                - name: OTEL_GENAI_EVALUATION_SAMPLING_RATE
+                  value: "1"
+              
+              # === Resource limits ===
+              resources:
+                requests:
+                  memory: "512Mi"
+                  cpu: "500m"
+                limits:
+                  memory: "1Gi"
+                  cpu: "1000m"
+

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/cronjob-traceloop.yaml

@@ -876,33 +876,10 @@ def main() -> None:
         )
         root_span.set_attribute("http.response.status_code", 200)
 
-    # Flush all telemetry providers before exit
-    print("\n🔄 Flushing telemetry data...")
-    
-    # Flush traces
     provider = trace.get_tracer_provider()
     if hasattr(provider, "force_flush"):
-        print("  → Flushing traces...")
-        provider.force_flush(timeout_millis=30000)  # 30 seconds
-    
-    # Flush logs
-    log_provider = _logs.get_logger_provider()
-    if hasattr(log_provider, "force_flush"):
-        print("  → Flushing logs...")
-        log_provider.force_flush(timeout_millis=30000)  # 30 seconds
-    
-    # Flush metrics
-    meter_provider = metrics.get_meter_provider()
-    if hasattr(meter_provider, "force_flush"):
-        print("  → Flushing metrics...")
-        meter_provider.force_flush(timeout_millis=30000)  # 30 seconds
-    
-    # Give batch processors time to complete final export operations
-    print("  → Waiting for final batch export...")
-    time.sleep(5)
-    
-    print("✅ Telemetry flush complete!\n")
-    
+        provider.force_flush()
+    time.sleep(300)
     if hasattr(provider, "shutdown"):
         provider.shutdown()