Skip to content

Commit 2579cbe

Browse files
adityamehraadityamehra
committed
add eval support in traceloop
verify commit
1 parent f0682ca commit 2579cbe

File tree

8 files changed

+1580
-87
lines changed

8 files changed

+1580
-87
lines changed

‎instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/.env.traceloop‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copy this file to `.env` and update values before running the sample.
2+
3+
# Required OpenAI API key
4+
OPENAI_API_KEY=sk-YOUR_API_KEY
5+
6+
# Optional: override default model (defaults to gpt-4.1)
7+
# OPENAI_MODEL=gpt-4.1
8+
9+
# OTLP exporter configuration (update for your collector)
10+
# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
11+
# OTEL_EXPORTER_OTLP_PROTOCOL=grpc
12+
13+
# Traces will use this service.name
14+
OTEL_SERVICE_NAME=opentelemetry-multi-agent-traceloop-translator-evals
15+
16+
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
17+
OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental
18+
OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true
19+
OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT_MODE=SPAN_AND_EVENT
20+
OTEL_INSTRUMENTATION_GENAI_EMITTERS=span_metric_event,splunk
21+
OTEL_INSTRUMENTATION_GENAI_EMITTERS_EVALUATION=replace-category:SplunkEvaluationResults
22+
OTEL_INSTRUMENTATION_GENAI_EVALS_RESULTS_AGGREGATION=true
23+
OTEL_INSTRUMENTATION_GENAI_DEBUG=true

‎instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/Dockerfile.traceloop‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Use Python 3.12 as base image
2+
FROM python:3.12-slim
3+
4+
# Set working directory
5+
WORKDIR /app
6+
7+
# Install system dependencies
8+
RUN apt-get update && apt-get install -y --no-install-recommends \
9+
git \
10+
&& rm -rf /var/lib/apt/lists/*
11+
12+
# Copy the entire instrumentation-genai and util directories to maintain package paths
13+
# Build context is at repo root to access both instrumentation-genai/ and util/
14+
COPY instrumentation-genai/opentelemetry-instrumentation-langchain /app/opentelemetry-instrumentation-langchain
15+
COPY util/opentelemetry-util-genai /app/opentelemetry-util-genai
16+
COPY util/opentelemetry-util-genai-traceloop-translator /app/opentelemetry-util-genai-traceloop-translator
17+
COPY util/opentelemetry-util-genai-evals /app/opentelemetry-util-genai-evals
18+
COPY util/opentelemetry-util-genai-evals-deepeval /app/opentelemetry-util-genai-evals-deepeval
19+
COPY util/opentelemetry-util-genai-emitters-splunk /app/opentelemetry-util-genai-emitters-splunk
20+
21+
# Set working directory to the example
22+
WORKDIR /app/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner
23+
24+
# Install Python dependencies from requirements.traceloop.txt (excluding local -e packages)
25+
# First, create a temporary requirements file without the local editable packages
26+
RUN grep -v "^-e \.\." requirements.traceloop.txt > /tmp/requirements_external.txt && \
27+
pip install --no-cache-dir -r /tmp/requirements_external.txt && \
28+
rm /tmp/requirements_external.txt
29+
30+
# Install the local packages in editable mode
31+
# The Traceloop translator will enable zero-code instrumentation via .pth file
32+
RUN cd /app/opentelemetry-util-genai && \
33+
pip install --no-cache-dir --no-deps -e . && \
34+
cd /app/opentelemetry-util-genai-evals && \
35+
pip install --no-cache-dir --no-deps -e . && \
36+
cd /app/opentelemetry-util-genai-evals-deepeval && \
37+
pip install --no-cache-dir --no-deps -e . && \
38+
cd /app/opentelemetry-util-genai-emitters-splunk && \
39+
pip install --no-cache-dir --no-deps -e . && \
40+
cd /app/opentelemetry-instrumentation-langchain && \
41+
pip install --no-cache-dir --no-deps -e . && \
42+
cd /app/opentelemetry-util-genai-traceloop-translator && \
43+
pip install --no-cache-dir --no-deps -e .
44+
45+
# Verify packages are installed correctly
46+
RUN python3 -c "from opentelemetry.util.genai.handler import get_telemetry_handler; print('✓ GenAI handler available')" && \
47+
python3 -c "from opentelemetry.util.genai.evals import create_evaluation_manager; print('✓ Evaluation manager available')" && \
48+
python3 -c "import opentelemetry.util.genai.emitters.splunk; print('✓ Splunk emitters available')" && \
49+
python3 -c "import opentelemetry.util.evaluator.deepeval; print('✓ Deepeval evaluator module available')" && \
50+
python3 -c "import deepeval; print('✓ Deepeval SDK installed')" && \
51+
python3 -c "from opentelemetry.util.genai.traceloop import enable_traceloop_translator; print('✓ Traceloop translator available')"
52+
53+
# Make the script executable
54+
RUN chmod +x main_traceloop.py
55+
56+
# Set default environment variables for OpenTelemetry
57+
ENV OTEL_PYTHON_LOG_CORRELATION=true \
58+
OTEL_PYTHON_LOG_LEVEL=info \
59+
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
60+
PYTHONUNBUFFERED=1
61+
62+
# Health check (optional)
63+
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
64+
CMD python3 -c "import sys; sys.exit(0)"
65+
66+
# Run the Traceloop version
67+
CMD ["python3", "main_traceloop.py"]
68+

‎instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/cronjob-tl.yaml‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
apiVersion: batch/v1
2+
kind: CronJob
3+
metadata:
4+
name: travel-planner-tl
5+
namespace: o11y-4-ai-admehra
6+
labels:
7+
app: travel-planner-tl
8+
component: telemetry
9+
annotations:
10+
description: "Multi-agent travel planner with Traceloop translator and GenAI evaluations (Deepeval telemetry disabled)"
11+
git-commit: "30c512d"
12+
spec:
13+
# Run every 30 minutes from 8 AM to 5 PM PST on weekdays (Monday-Friday)
14+
schedule: "*/30 8-17 * * 1-5"
15+
timeZone: "America/Los_Angeles"
16+
suspend: false
17+
18+
# Keep last 3 successful and 1 failed job for debugging
19+
successfulJobsHistoryLimit: 3
20+
failedJobsHistoryLimit: 1
21+
22+
jobTemplate:
23+
metadata:
24+
labels:
25+
app: travel-planner-tl
26+
component: telemetry
27+
spec:
28+
template:
29+
metadata:
30+
labels:
31+
app: travel-planner-tl
32+
component: telemetry
33+
spec:
34+
restartPolicy: OnFailure
35+
36+
containers:
37+
- name: travel-planner-traceloop
38+
# Multi-platform image (amd64, arm64) with git commit hash tag
39+
image: admehra621/travel-planner-tl:30c512d
40+
imagePullPolicy: Always
41+
42+
env:
43+
# === GenAI Semantic Conventions (REQUIRED) ===
44+
- name: OTEL_SEMCONV_STABILITY_OPT_IN
45+
value: "gen_ai_latest_experimental"
46+
47+
# === OpenTelemetry Resource Attributes ===
48+
- name: OTEL_RESOURCE_ATTRIBUTES
49+
value: "deployment.environment=o11y-inframon-ai,git.commit.id=30c512d"
50+
51+
# === Service name for telemetry ===
52+
- name: OTEL_SERVICE_NAME
53+
value: "travel-planner-tl"
54+
55+
# === OpenAI Configuration ===
56+
- name: OPENAI_API_KEY
57+
valueFrom:
58+
secretKeyRef:
59+
name: openai-credentials
60+
key: api-key
61+
62+
- name: OPENAI_MODEL
63+
value: "gpt-4o-mini"
64+
65+
# === Deepeval Telemetry Opt-Out ===
66+
- name: DEEPEVAL_TELEMETRY_OPT_OUT
67+
value: "1"
68+
69+
# === GenAI Content Capture ===
70+
- name: OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT
71+
value: "true"
72+
73+
- name: OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT_MODE
74+
value: "SPAN_AND_EVENT"
75+
76+
# === GenAI Emitters Configuration ===
77+
- name: OTEL_INSTRUMENTATION_GENAI_EMITTERS
78+
value: "span_metric_event,splunk"
79+
80+
- name: OTEL_INSTRUMENTATION_GENAI_EMITTERS_EVALUATION
81+
value: "replace-category:SplunkEvaluationResults"
82+
83+
# === Evaluation Settings ===
84+
# All 5 default evaluations enabled (bias, toxicity, relevance, hallucination, sentiment)
85+
- name: OTEL_INSTRUMENTATION_GENAI_EVALS_RESULTS_AGGREGATION
86+
value: "true"
87+
88+
# === GenAI Debug Flags (disabled for production) ===
89+
# - name: OTEL_GENAI_EVAL_DEBUG_SKIPS
90+
# value: "true"
91+
# - name: OTEL_GENAI_EVAL_DEBUG_EACH
92+
# value: "true"
93+
# - name: OTEL_INSTRUMENTATION_GENAI_DEBUG
94+
# value: "true"
95+
96+
# === OpenTelemetry Logs Exporter ===
97+
- name: OTEL_LOGS_EXPORTER
98+
value: "otlp"
99+
100+
# === Get the host IP for Splunk OTEL agent ===
101+
- name: SPLUNK_OTEL_AGENT
102+
valueFrom:
103+
fieldRef:
104+
fieldPath: status.hostIP
105+
106+
# === OpenTelemetry OTLP endpoint using Splunk agent ===
107+
- name: OTEL_EXPORTER_OTLP_ENDPOINT
108+
value: "http://$(SPLUNK_OTEL_AGENT):4317"
109+
110+
# === OTLP Protocol (grpc) ===
111+
- name: OTEL_EXPORTER_OTLP_PROTOCOL
112+
value: "grpc"
113+
114+
# === Exclude health check URLs ===
115+
- name: OTEL_PYTHON_EXCLUDED_URLS
116+
value: "^(https?://)?[^/]+(/)?$"
117+
118+
# === Enable Python logging auto instrumentation ===
119+
- name: OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED
120+
value: "true"
121+
122+
# === Enable log correlation ===
123+
- name: OTEL_PYTHON_LOG_CORRELATION
124+
value: "true"
125+
126+
# === Enable LangChain content capture ===
127+
- name: OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT
128+
value: "true"
129+
130+
# === Enable Splunk profiler ===
131+
- name: SPLUNK_PROFILER_ENABLED
132+
value: "true"
133+
134+
# === Unbuffered Python output ===
135+
- name: PYTHONUNBUFFERED
136+
value: "1"
137+
138+
# === GenAI evaluation sampling rate ===
139+
- name: OTEL_GENAI_EVALUATION_SAMPLING_RATE
140+
value: "1"
141+
142+
# === Resource limits ===
143+
resources:
144+
requests:
145+
memory: "512Mi"
146+
cpu: "500m"
147+
limits:
148+
memory: "1Gi"
149+
cpu: "1000m"
150+

0 commit comments

Comments
 (0)