feat: tweak pdf parser for corner cases and add 120s demo

Author: YoungVor

examples/fenic_in_120_seconds/18_pdf_processing.ipynb

@@ -24,8 +24,9 @@ class ParsePDF(BaseSingleColumnFilePathOperator[str, str]):
     """Operator for parsing PDF files using language models with PDF parsing capabilities."""
     SYSTEM_PROMPT = jinja2.Template(dedent("""\
         Transcribe the main content of this PDF document to clean, well-formatted markdown.
-         - Output should be raw markdown, don't surround in code fences or backticks.
-         - Preserve the structure, formatting, headings, lists, and any tables to the best of your ability
+         - Output should be raw markdown, don't surround the whole output in code fences or backticks.
+         - For each topic, create a markdown heading. For key terms, use bold text.
+         - Preserve the structure, formatting, headings, lists, table of contents, and any tables using markdown syntax.
          - Format tables as github markdown tables, however:
              - for table headings, immediately add ' |' after the table heading
         {% if multiple_pages %}

src/fenic/_backends/local/semantic_operators/parse_pdf.py

@@ -90,10 +90,9 @@ async def make_single_request(
             common_params: dict[str, Any] = {
                 "model": self._model,
                 "messages": convert_messages(request.messages),
+                "max_completion_tokens": self._get_max_output_token_request_limit(request),
                 "n": 1,
             }
-            if request.max_completion_tokens:
-                common_params.update({"max_completion_tokens": request.max_completion_tokens + profile_configuration.expected_additional_reasoning_tokens})
             if request.temperature:
                 common_params.update({"temperature": request.temperature})
 

src/fenic/_inference/common_openai/openai_chat_completions_core.py

@@ -27,6 +27,7 @@
     TransientException,
 )
 from fenic._inference.rate_limit_strategy import (
+    MAX_OUTPUT_TOKENS_PER_PARSE_PDF_REQUEST,
     TokenEstimate,
     UnifiedTokenRateLimitStrategy,
 )
@@ -176,16 +177,14 @@ async def make_single_request(
         """
 
         profile_config = self._profile_manager.get_profile_by_name(request.model_profile)
-        max_output_tokens = self._get_max_output_token_request_limit(request)
-
         generation_config: GenerateContentConfigDict = {
             "temperature": request.temperature,
             "response_logprobs": request.top_logprobs is not None,
             "logprobs": request.top_logprobs,
+            "max_output_tokens": self._get_max_output_token_request_limit(request),
             "system_instruction": request.messages.system,
         }
-        if max_output_tokens is not None:
-            generation_config["max_output_tokens"] = max_output_tokens
+
         generation_config.update(profile_config.additional_generation_config)
         if request.structured_output is not None:
             generation_config.update(
@@ -342,14 +341,15 @@ def _estimate_output_tokens(self, request: FenicCompletionsRequest) -> int:
     def _get_max_output_token_request_limit(self, request: FenicCompletionsRequest) -> Optional[int]:
         """Get the upper limit of output tokens for a request.
 
-        If max_completion_tokens is not set, don't apply a limit and return None.
+        For file parsing requests, use a guardrail limit of 8192 tokens (the lowest output limit of a VLM model we support).
 
         Include the thinking token budget with a safety margin."""
         max_output_tokens = request.max_completion_tokens or 0
         if request.max_completion_tokens is None and request.messages.user_file:
             # Guardrail to ensure the model uses a sane amount of output tokens.
+            # Note: we can't use our token estimation because the pdf could be empty, or have only images (scans).
             # TODO(DY): the semantic operator should dictate how the file affects the token estimate
-            max_output_tokens = self.token_counter.count_file_output_tokens(request.messages) * 2
+            max_output_tokens = MAX_OUTPUT_TOKENS_PER_PARSE_PDF_REQUEST
         return max_output_tokens + self._get_expected_additional_reasoning_tokens(request)
 
     def _get_expected_additional_reasoning_tokens(self, request: FenicCompletionsRequest) -> int:

src/fenic/_inference/google/gemini_native_chat_completions_client.py

@@ -15,6 +15,7 @@
 )
 from fenic._inference.openai.openai_provider import OpenAIModelProvider
 from fenic._inference.rate_limit_strategy import (
+    MAX_OUTPUT_TOKENS_PER_PARSE_PDF_REQUEST,
     RateLimitStrategy,
     TokenEstimate,
 )
@@ -137,8 +138,9 @@ def _get_max_output_token_request_limit(self, request: FenicCompletionsRequest)
         max_output_tokens = request.max_completion_tokens or 0
         if request.max_completion_tokens is None and request.messages.user_file:
             # Guardrail to ensure the model uses a sane amount of output tokens.
+            # Note: we can't use our token estimation because the pdf could be empty, or have only images (scans).
             # TODO(DY): the semantic operator should dictate how the file affects the token estimate
-            max_output_tokens = self.token_counter.count_file_output_tokens(request.messages) * 2
+            max_output_tokens = MAX_OUTPUT_TOKENS_PER_PARSE_PDF_REQUEST
         return max_output_tokens + self._get_expected_additional_reasoning_tokens(request)
 
     def _get_expected_additional_reasoning_tokens(self, request: FenicCompletionsRequest) -> int:

src/fenic/_inference/openai/openai_batch_chat_completions_client.py

@@ -21,6 +21,7 @@
 )
 from fenic._inference.openrouter.openrouter_provider import OpenRouterModelProvider
 from fenic._inference.rate_limit_strategy import (
+    MAX_OUTPUT_TOKENS_PER_PARSE_PDF_REQUEST,
     AdaptiveBackoffRateLimitStrategy,
     RateLimitStrategy,
     TokenEstimate,
@@ -268,17 +269,15 @@ def _estimate_output_tokens(self, request: FenicCompletionsRequest) -> int:
     def _get_max_output_token_request_limit(self, request: FenicCompletionsRequest) -> int:
         """Get the upper limit of output tokens for a request.
 
-        If max_completion_tokens is not set, don't apply a limit and return None.
+        For file parsing requests, use a guardrail limit of 8192 tokens (the lowest output limit of a VLM model we support).
 
         Include the thinking token budget with a safety margin."""
-        if request.max_completion_tokens:
-            max_output_tokens = request.max_completion_tokens
-        elif request.messages.user_file:
+        max_output_tokens = request.max_completion_tokens
+        if request.max_completion_tokens is None and request.messages.user_file:
             # Guardrail to ensure the model uses a sane amount of output tokens.
-            if self._google_token_counter:
-                max_output_tokens = self._google_token_counter.count_file_output_tokens(messages=request.messages) * 2
-            else:
-                max_output_tokens = self.token_counter.count_file_output_tokens(messages=request.messages) * 2
+            # Note: we can't use our token estimation because the pdf could be empty, or have only images (scans).
+            # TODO(DY): the semantic operator should dictate how the file affects the token estimate
+            max_output_tokens = MAX_OUTPUT_TOKENS_PER_PARSE_PDF_REQUEST
         return max_output_tokens + self._get_expected_additional_reasoning_tokens(request)
 
     def _estimate_input_tokens(self, request: FenicCompletionsRequest) -> int:

src/fenic/_inference/openrouter/openrouter_batch_chat_completions_client.py

@@ -10,7 +10,7 @@
 
 logger = logging.getLogger(__name__)
 
-
+MAX_OUTPUT_TOKENS_PER_PARSE_PDF_REQUEST = 8192
 @dataclass
 class TokenEstimate:
     input_tokens: int = 0