[VPC ChatCompletion] Add functionality for structured outputs per-field scoring (#127)

huiwengoh · web-flow · commit db6eeefbeea1 · 2025-09-30T21:30:14.000-04:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.1.36] - 2025-09-30
+
+## Added
+
+- Extend `TLMResponses` to work for OpenAI-built-in tools
+- Add per-field scoring functionality for structured outputs responses in VPC ChatCompletion module
+
 ## [1.1.35] - 2025-09-25
 
 ### Added 
@@ -367,7 +374,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Release of the Cleanlab TLM Python client.
 
-[Unreleased]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.35...HEAD
+[Unreleased]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.36...HEAD
+[1.1.36]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.35...v1.1.36
 [1.1.35]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.34...v1.1.35
 [1.1.34]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.33...v1.1.34
 [1.1.33]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.32...v1.1.33
diff --git a/src/cleanlab_tlm/__about__.py b/src/cleanlab_tlm/__about__.py
@@ -1,2 +1,2 @@
 # SPDX-License-Identifier: MIT
-__version__ = "1.1.35"
+__version__ = "1.1.36"
diff --git a/src/cleanlab_tlm/utils/chat_completions.py b/src/cleanlab_tlm/utils/chat_completions.py
@@ -6,7 +6,6 @@
 """
 
 import asyncio
-import json
 from typing import TYPE_CHECKING, Any, Optional, Union, cast
 
 from cleanlab_tlm.internal.api.api import tlm_chat_completions_score
@@ -18,6 +17,7 @@
 from cleanlab_tlm.internal.types import TLMQualityPreset
 from cleanlab_tlm.tlm import TLM, TLMOptions, TLMResponse, TLMScore
 from cleanlab_tlm.utils.chat import _form_prompt_chat_completions_api, form_response_string_chat_completions
+from cleanlab_tlm.utils.per_field_score_utils import _get_untrustworthy_fields
 
 if TYPE_CHECKING:
     from openai.types.chat import ChatCompletion, ChatCompletionMessage
@@ -251,75 +251,13 @@ def get_untrustworthy_fields(
         Returns:
             list[str]: The fields of the response that are considered untrustworthy by TLM
         """
-        try:
-            from openai.types.chat import ChatCompletion
-        except ImportError as e:
-            raise ImportError(
-                f"OpenAI is required to use the {self.__class__.__name__} class. Please install it with `pip install openai`."
-            ) from e
-
-        if isinstance(tlm_result, dict):
-            if response is None:
-                raise ValueError("'response' is required when tlm_result is a TLMScore object")
-
-            tlm_metadata = tlm_result
-            response_text = response.choices[0].message.content or "{}"
-
-        elif isinstance(tlm_result, ChatCompletion):
-            if getattr(tlm_result, "tlm_metadata", None) is None:
-                raise ValueError("tlm_result must contain tlm_metadata.")
-
-            tlm_metadata = tlm_result.tlm_metadata  # type: ignore
-            response_text = tlm_result.choices[0].message.content or "{}"
-
-        else:
-            raise TypeError("tlm_result must be a TLMScore or ChatCompletion object.")
-
-        if "per_field_score" not in tlm_metadata.get("log", {}):
-            raise ValueError(
-                "`per_field_score` is not present in the log.\n"
-                "`get_untrustworthy_fields()` can only be called scoring structured outputs responses and specifying "
-                "`per_field_score` in the `log` option for TLM."
-            )
-
-        try:
-            so_response = json.loads(response_text)
-        except Exception:
-            raise ValueError(
-                "The LLM response must be a valid JSON output (use `response_format` to specify the output format)"
-            )
-
-        per_field_score = tlm_metadata["log"]["per_field_score"]
-        per_score_details = []
-
-        for key, value in per_field_score.items():
-            score = value["score"]
-            if float(score) < threshold:
-                key_details = {
-                    "response": so_response[key],
-                    "score": score,
-                    "explanation": value["explanation"],
-                }
-                per_score_details.append({key: key_details})
-
-        per_score_details.sort(key=lambda x: next(iter(x.values()))["score"])
-        untrustworthy_fields = [next(iter(item.keys())) for item in per_score_details]
-
-        if display_details:
-            if len(untrustworthy_fields) == 0:
-                print("No untrustworthy fields found")
-
-            else:
-                print(f"Untrustworthy fields: {untrustworthy_fields}\n")
-                for item in per_score_details:
-                    print(f"Field: {next(iter(item.keys()))}")
-                    details = next(iter(item.values()))
-                    print(f"Response: {details['response']}")
-                    print(f"Score: {details['score']}")
-                    print(f"Explanation: {details['explanation']}")
-                    print()
-
-        return untrustworthy_fields
+        return _get_untrustworthy_fields(
+            response=response,
+            tlm_result=tlm_result,
+            threshold=threshold,
+            display_details=display_details,
+            class_name=self.__class__.__name__,
+        )
 
     @staticmethod
     def _get_response_message(response: "ChatCompletion") -> "ChatCompletionMessage":
diff --git a/src/cleanlab_tlm/utils/per_field_score_utils.py b/src/cleanlab_tlm/utils/per_field_score_utils.py
@@ -0,0 +1,85 @@
+import json
+from typing import TYPE_CHECKING, Optional, Union
+
+from cleanlab_tlm.tlm import TLMScore
+
+if TYPE_CHECKING:
+    from openai.types.chat import ChatCompletion
+
+
+def _get_untrustworthy_fields(
+    response: Optional["ChatCompletion"],
+    tlm_result: Union[TLMScore, "ChatCompletion"],
+    threshold: float,
+    display_details: bool,
+    class_name: str,
+) -> list[str]:
+    try:
+        from openai.types.chat import ChatCompletion
+    except ImportError as e:
+        raise ImportError(
+            f"OpenAI is required to use the {class_name} class. Please install it with `pip install openai`."
+        ) from e
+
+    if isinstance(tlm_result, dict):
+        if response is None:
+            raise ValueError("'response' is required when tlm_result is a TLMScore object")
+
+        tlm_metadata = tlm_result
+        response_text = response.choices[0].message.content or "{}"
+
+    elif isinstance(tlm_result, ChatCompletion):
+        if getattr(tlm_result, "tlm_metadata", None) is None:
+            raise ValueError("tlm_result must contain tlm_metadata.")
+
+        tlm_metadata = tlm_result.tlm_metadata  # type: ignore
+        response_text = tlm_result.choices[0].message.content or "{}"
+
+    else:
+        raise TypeError("tlm_result must be a TLMScore or ChatCompletion object.")
+
+    if "per_field_score" not in tlm_metadata.get("log", {}):
+        raise ValueError(
+            "`per_field_score` is not present in the log.\n"
+            "`get_untrustworthy_fields()` can only be called scoring structured outputs responses and specifying "
+            "`per_field_score` in the `log` option for TLM."
+        )
+
+    try:
+        so_response = json.loads(response_text)
+    except Exception:
+        raise ValueError(
+            "The LLM response must be a valid JSON output (use `response_format` to specify the output format)"
+        )
+
+    per_field_score = tlm_metadata["log"]["per_field_score"]
+    per_score_details = []
+
+    for key, value in per_field_score.items():
+        score = value["score"]
+        if float(score) < threshold:
+            key_details = {
+                "response": so_response[key],
+                "score": score,
+                "explanation": value["explanation"],
+            }
+            per_score_details.append({key: key_details})
+
+    per_score_details.sort(key=lambda x: next(iter(x.values()))["score"])
+    untrustworthy_fields = [next(iter(item.keys())) for item in per_score_details]
+
+    if display_details:
+        if len(untrustworthy_fields) == 0:
+            print("No untrustworthy fields found")
+
+        else:
+            print(f"Untrustworthy fields: {untrustworthy_fields}\n")
+            for item in per_score_details:
+                print(f"Field: {next(iter(item.keys()))}")
+                details = next(iter(item.values()))
+                print(f"Response: {details['response']}")
+                print(f"Score: {details['score']}")
+                print(f"Explanation: {details['explanation']}")
+                print()
+
+    return untrustworthy_fields
diff --git a/src/cleanlab_tlm/utils/vpc/chat_completions.py b/src/cleanlab_tlm/utils/vpc/chat_completions.py
@@ -11,19 +11,19 @@
 from __future__ import annotations
 
 import os
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Optional, Union, cast
 
 import requests
 
 from cleanlab_tlm.internal.base import BaseTLM
 from cleanlab_tlm.internal.constants import _VALID_TLM_QUALITY_PRESETS_CHAT_COMPLETIONS
+from cleanlab_tlm.tlm import TLMScore
+from cleanlab_tlm.utils.per_field_score_utils import _get_untrustworthy_fields
 from cleanlab_tlm.utils.vpc.tlm import VPCTLMOptions
 
 if TYPE_CHECKING:
     from openai.types.chat import ChatCompletion
 
-    from cleanlab_tlm.internal.types import JSONDict
-
 
 class TLMChatCompletion(BaseTLM):
     """
@@ -74,7 +74,7 @@ def score(
         *,
         response: ChatCompletion,
         **openai_kwargs: Any,
-    ) -> JSONDict:
+    ) -> TLMScore:
         """Score the trustworthiness of an OpenAI ChatCompletion response.
 
         Args:
@@ -84,12 +84,22 @@ def score(
         Returns:
             TLMScore: A dict containing the trustworthiness score and optional logs
         """
+        try:
+            from openai.lib._parsing._completions import type_to_response_format_param
+        except ImportError as e:
+            raise ImportError(
+                f"OpenAI is required to use the {self.__class__.__name__} class. Please install it with `pip install openai`."
+            ) from e
+
         if (base_url := os.environ.get("BASE_URL")) is None:
             raise ValueError("BASE_URL is not set. Please set it in the environment variables.")
 
         # replace the model used for scoring with the specified model in options
         openai_kwargs["model"] = self._options["model"]
 
+        if "response_format" in openai_kwargs:
+            openai_kwargs["response_format"] = type_to_response_format_param(openai_kwargs["response_format"])
+
         res = requests.post(
             f"{base_url}/chat/score",
             json={
@@ -112,7 +122,49 @@ def score(
 
         res_json = res.json()
         tlm_result = {"trustworthiness_score": res_json["tlm_metadata"]["trustworthiness_score"]}
-        if explanation := res_json["tlm_metadata"].get("log", {}).get("explanation"):
-            tlm_result["log"] = {"explanation": explanation}
 
-        return tlm_result
+        if self._return_log:
+            log = {}
+
+            log_options = cast(list[str], self._options.get("log", []))
+            if "explanation" in log_options:
+                explanation = res_json["tlm_metadata"].get("log", {}).get("explanation")
+                log["explanation"] = explanation
+
+            if "per_field_score" in log_options:
+                per_field_score = res_json["tlm_metadata"].get("log", {}).get("per_field_score")
+                log["per_field_score"] = per_field_score
+
+            tlm_result["log"] = log
+
+        return cast(TLMScore, tlm_result)
+
+    def get_untrustworthy_fields(
+        self,
+        *,
+        response: Optional[ChatCompletion] = None,
+        tlm_result: Union[TLMScore, ChatCompletion],
+        threshold: float = 0.8,
+        display_details: bool = True,
+    ) -> list[str]:
+        """Gets the fields of a structured output response that are considered untrustworthy by TLM.
+        Only works for responses that are valid JSON objects (uses `response_format` to specify the output format).
+        Prints detailed information about the untrustworthy fields if `display_details` is True.
+
+        Args:
+            response (ChatCompletion): The OpenAI ChatCompletion response object to evaluate
+            tlm_result (TLMScore | ChatCompletion): The result object from a previous TLM call
+            threshold (float): The threshold for considering a field untrustworthy
+            display_details (bool): Whether to display detailed information about the untrustworthy fields
+
+        Returns:
+            list[str]: The fields of the response that are considered untrustworthy by TLM
+        """
+
+        return _get_untrustworthy_fields(
+            response=response,
+            tlm_result=tlm_result,
+            threshold=threshold,
+            display_details=display_details,
+            class_name=self.__class__.__name__,
+        )

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`# SPDX-License-Identifier: MIT`
`2`		`-__version__ = "1.1.35"`
	`2`	`+__version__ = "1.1.36"`