Skip evaluations for tool calls llm and errored llm invocations #39 (#40)

Author: wrisa

instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py

@@ -426,9 +426,24 @@ def on_llm_end(
         if generations and generations[0] and generations[0][0].message:
             content = getattr(generations[0][0].message, "content", None)
         if content is not None:
-            inv.output_messages = [
-                OutputMessage(role="assistant", parts=[Text(content=_safe_str(content))], finish_reason="stop")
-            ]
+            finish_reason = generations[0][0].generation_info.get("finish_reason") if generations[0][
+                0].generation_info else None
+            if finish_reason == "tool_calls":
+                inv.output_messages = [
+                    OutputMessage(
+                        role="assistant",
+                        parts=["ToolCall"],
+                        finish_reason=finish_reason or "tool_calls",
+                    )
+                ]
+            else:
+                inv.output_messages = [
+                    OutputMessage(
+                        role="assistant",
+                        parts=[Text(content=_safe_str(content))],
+                        finish_reason=finish_reason or "stop",
+                    )
+                ]
         llm_output = getattr(response, "llm_output", {}) or {}
         usage = llm_output.get("usage") or llm_output.get("token_usage") or {}
         inv.input_tokens = usage.get("prompt_tokens")

util/opentelemetry-util-genai-evals/src/opentelemetry/util/genai/evals/manager.py

@@ -35,6 +35,9 @@
 )
 from .normalize import is_tool_only_llm
 from .registry import get_default_metrics, get_evaluator, list_evaluators
+from opentelemetry.semconv.attributes import (
+    error_attributes as ErrorAttributes,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -108,8 +111,24 @@ def on_completion(self, invocation: GenAI) -> None:
         if not self.has_evaluators:
             return
 
+        offer: bool = True
         if invocation.sample_for_evaluation:
             self.offer(invocation)
+            # Do not evaluate if llm invocation is for tool invocation because it will not have output message for evaluations tests case.
+            if isinstance(invocation, LLMInvocation):
+                msgs = getattr(invocation, "output_messages", [])
+                if msgs:
+                    first = msgs[0]
+                    if first.parts and first.parts[0] == "ToolCall" and first.finish_reason == "tool_calls":
+                        offer = False
+
+            # Do not evaluate if error
+            error = invocation.attributes.get(ErrorAttributes.ERROR_TYPE)
+            if error:
+                offer = False
+
+            if offer:
+                self.offer(invocation)
 
     # Public API ---------------------------------------------------------
     def offer(self, invocation: GenAI) -> None:

util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py

@@ -113,6 +113,9 @@ def genai_debug_log(*_args: Any, **_kwargs: Any) -> None:  # type: ignore
     OTEL_INSTRUMENTATION_GENAI_DISABLE_DEFAULT_COMPLETION_CALLBACKS,
 )
 from opentelemetry.sdk.trace.sampling import Decision, TraceIdRatioBased
+from opentelemetry.semconv.attributes import (
+    error_attributes as ErrorAttributes,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -920,6 +923,7 @@ def fail_by_run_id(self, run_id: Any, error: Error) -> None:
         entity = self.get_entity(run_id)
         if entity is None:
             return
+        entity.attributes.update({ErrorAttributes.ERROR_TYPE: getattr(error.type, "__qualname__", str(error.type))})
         if isinstance(entity, Workflow):
             self.fail_workflow(entity, error)
         elif isinstance(entity, (AgentCreation, AgentInvocation)):