Make prompt/score methods capture exceptions by default (#51)

Author: huiwengoh

CHANGELOG.md

@@ -7,8 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.1.0] - 2025-04-21
+
+### Changed 
+
+- All `.prompt()` / `.get_trustworthiness_score()` / `.generate()` / `.score()` methods will now catch any errors and return `null` values alongside a log of why the exception occurred
+- `try_` methods are deprecated and will share the same functionality as the "non-try" methods
+
 ## [1.0.22] - 2025-04-18
 
+### Added 
+
 - Update `response_helpfulness` default criteria
 
 ## [1.0.21] - 2025-04-17
@@ -125,7 +134,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Release of the Cleanlab TLM Python client.
 
-[Unreleased]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.0.23...HEAD
+[Unreleased]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.0...HEAD
+[1.1.0]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.0.23...v1.1.0
 [1.0.23]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.0.22...v1.0.23
 [1.0.22]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.0.21...v1.0.22
 [1.0.21]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.0.20...v1.0.21

src/cleanlab_tlm/__about__.py

@@ -1,2 +1,2 @@
 # SPDX-License-Identifier: MIT
-__version__ = "1.0.23"
+__version__ = "1.1.0"

src/cleanlab_tlm/internal/exception_handling.py

@@ -58,7 +58,7 @@ def decorator(
         async def wrapper(*args: Any, **kwargs: Any) -> ResponseT:
             capture_exceptions = kwargs.get("capture_exceptions", False)
             batch_index = kwargs.get("batch_index")
-            evals = kwargs.get("evals", [])
+            evals = getattr(args[0], "_evals", [])
             try:
                 return await func(*args, **kwargs)
             except asyncio.TimeoutError:
@@ -162,7 +162,9 @@ def _handle_exception(
         error_log = {"error": {"message": error_message, "retryable": retryable}}
 
         # Helper function to create evaluation metrics dictionary
-        def create_eval_metrics(include_response: bool = False) -> dict[str, Union[dict[str, Any], None]]:
+        def create_eval_metrics(
+            include_response: bool = False,
+        ) -> dict[str, Union[dict[str, Any], None]]:
             result: dict[str, Union[dict[str, Any], None]] = {
                 "trustworthiness": {
                     "score": None,
@@ -206,9 +208,4 @@ def create_eval_metrics(include_response: bool = False) -> dict[str, Union[dict[
 
         raise ValueError(f"Unsupported response type: {response_type}")
 
-    if len(e.args) > 0 and response_type != RAG_GENERATE_RESPONSE_TYPE and response_type != RAG_SCORE_RESPONSE_TYPE:
-        additional_message = "Consider using `TLM.try_prompt()` or `TLM.try_get_trustworthiness_score()` to gracefully handle errors and preserve partial results. For large datasets, consider also running it on multiple smaller batches."
-        new_args = (str(e.args[0]) + "\n" + additional_message,) + e.args[1:]
-        raise type(e)(*new_args)
-
     raise e  # in the case where the error has no message/args

src/cleanlab_tlm/internal/validation.py

@@ -35,16 +35,6 @@ def validate_tlm_prompt(prompt: Union[str, Sequence[str]]) -> None:
         )
 
 
-def validate_tlm_try_prompt(prompt: Sequence[str]) -> None:
-    if isinstance(prompt, str):
-        raise ValidationError("Invalid type str, prompt must be a list/iterable of strings.")
-
-    if isinstance(prompt, Sequence) and any(not isinstance(p, str) for p in prompt):
-        raise ValidationError(
-            "Some items in prompt are of invalid types, all items in the prompt list must be of type str."
-        )
-
-
 def validate_tlm_prompt_response(prompt: Union[str, Sequence[str]], response: Union[str, Sequence[str]]) -> None:
     if isinstance(prompt, str):
         if not isinstance(response, str):
@@ -74,24 +64,6 @@ def validate_tlm_prompt_response(prompt: Union[str, Sequence[str]], response: Un
             )
 
 
-def validate_try_tlm_prompt_response(prompt: Sequence[str], response: Sequence[str]) -> None:
-    if isinstance(prompt, str):
-        raise ValidationError("Invalid type str, prompt must be a list/iterable of strings.")
-
-    if isinstance(prompt, Sequence):
-        if len(prompt) != len(response):
-            raise ValidationError("Length of the prompt and response lists must match.")
-
-        if any(not isinstance(p, str) for p in prompt):
-            raise ValidationError(
-                "Some items in prompt are of invalid types, all items in the prompt list must be of type str."
-            )
-        if any(not isinstance(r, str) for r in response):
-            raise ValidationError(
-                "Some items in response are of invalid types, all items in the response list must be of type str."
-            )
-
-
 def validate_tlm_options(options: Any, support_custom_eval_criteria: bool = True) -> None:
     from cleanlab_tlm.tlm import TLMOptions