[Renderer] Introduce Renderer for processing chat messages (using RendererConfig)

benchmarks/backend_request_func.py

@@ -620,7 +620,7 @@ def get_tokenizer(
         kwargs["use_fast"] = False
     if tokenizer_mode == "mistral":
         try:
-            from vllm.tokenizers import MistralTokenizer
+            from vllm.tokenizers.mistral import MistralTokenizer
         except ImportError as e:
             raise ImportError(
                 "MistralTokenizer requires vllm package.\n"

tests/entrypoints/openai/test_chat_template.py

@@ -3,8 +3,9 @@
 
 import pytest
 
-from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template
+from vllm.entrypoints.chat_utils import load_chat_template
 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
+from vllm.renderers.hf import safe_apply_chat_template
 from vllm.tokenizers import get_tokenizer
 
 from ...models.registry import HF_EXAMPLE_MODELS
@@ -125,14 +126,15 @@ def test_get_gen_prompt(
     )
 
     # Call the function and get the result
-    result = apply_hf_chat_template(
-        tokenizer=tokenizer,
-        conversation=mock_request.messages,
-        chat_template=mock_request.chat_template or template_content,
-        renderer_config=renderer_config,
+    result = safe_apply_chat_template(
+        renderer_config,
+        tokenizer,
+        mock_request.messages,
         tools=None,
+        chat_template=mock_request.chat_template or template_content,
         add_generation_prompt=mock_request.add_generation_prompt,
         continue_final_message=mock_request.continue_final_message,
+        tokenize=False,
     )
 
     # Test assertion

tests/entrypoints/openai/test_serving_engine.py

@@ -10,7 +10,7 @@
 from vllm.config import ModelConfig, RendererConfig
 from vllm.entrypoints.openai.serving_engine import OpenAIServing
 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
-from vllm.tokenizers import MistralTokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 
 
 @pytest.fixture()

tests/entrypoints/test_chat_utils.py

@@ -29,7 +29,8 @@
     encode_image_base64,
     encode_video_base64,
 )
-from vllm.tokenizers import MistralTokenizer, get_tokenizer
+from vllm.tokenizers import get_tokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 from vllm.utils.serial_utils import tensor2base64
 
 from ..models.registry import HF_EXAMPLE_MODELS

tests/models/language/generation/test_mistral.py

@@ -10,7 +10,7 @@
     MistralToolParser,
 )
 from vllm.sampling_params import SamplingParams
-from vllm.tokenizers import MistralTokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 
 from ...utils import check_logprobs_close
 

tests/models/multimodal/generation/test_voxtral.py

@@ -9,7 +9,7 @@
 from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
 from mistral_common.protocol.instruct.messages import UserMessage
 
-from vllm.tokenizers import MistralTokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 
 from ....conftest import AudioTestAssets
 from ....utils import RemoteOpenAIServer

tests/models/multimodal/processing/test_common.py

@@ -22,10 +22,8 @@
 from vllm.multimodal.cache import MultiModalProcessorOnlyCache
 from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal
 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
-from vllm.tokenizers import (
-    MistralTokenizer,
-    TokenizerLike,
-)
+from vllm.tokenizers import TokenizerLike
+from vllm.tokenizers.mistral import MistralTokenizer
 
 from ....multimodal.utils import random_audio, random_image, random_video
 from ...registry import (

tests/test_inputs.py

@@ -7,7 +7,7 @@
 from vllm.inputs import zip_enc_dec_prompts
 from vllm.inputs.parse import parse_raw_prompts
 from vllm.inputs.preprocess import InputPreprocessor
-from vllm.tokenizers import init_tokenizer_from_config
+from vllm.renderers import RendererRegistry
 
 pytestmark = pytest.mark.cpu_test
 
@@ -109,10 +109,11 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
 def test_preprocessor_always_mm_code_path(model_id, prompt):
     model_config = ModelConfig(model=model_id)
     renderer_config = RendererConfig(model_config=model_config)
-    tokenizer = init_tokenizer_from_config(renderer_config)
-    input_preprocessor = InputPreprocessor(renderer_config, tokenizer)
+    renderer = RendererRegistry.get_renderer(renderer_config)
+    input_preprocessor = InputPreprocessor(renderer_config, renderer)
 
     # HF processor adds sep token
+    tokenizer = renderer.get_tokenizer()
     sep_token_id = tokenizer.vocab[tokenizer.sep_token]
 
     processed_inputs = input_preprocessor.preprocess(prompt)

tests/tokenizers_/test_registry.py

@@ -43,7 +43,7 @@ def is_fast(self) -> bool:
 def test_customized_tokenizer():
     TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
 
-    tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer", "abc")
+    tokenizer = TokenizerRegistry.init_tokenizer("test_tokenizer", "abc")
     assert isinstance(tokenizer, TestTokenizer)
     assert tokenizer.path_or_repo_id == "abc"
     assert tokenizer.bos_token_id == 0

tests/v1/engine/test_process_multi_modal_uuids.py

@@ -13,6 +13,7 @@
     RendererConfig,
     VllmConfig,
 )
+from vllm.renderers.terratorch import TerratorchRenderer
 from vllm.sampling_params import SamplingParams
 from vllm.v1.engine import input_processor as input_processor_mod
 from vllm.v1.engine.input_processor import InputProcessor
@@ -59,7 +60,6 @@ def _mock_input_processor(
 
     renderer_config = RendererConfig(
         model_config=model_config,
-        tokenizer="dummy",
         skip_tokenizer_init=True,
     )
 
@@ -70,7 +70,7 @@ def _mock_input_processor(
         device_config=DeviceConfig(device="cpu"),
     )
 
-    return InputProcessor(vllm_config, tokenizer=None)
+    return InputProcessor(vllm_config, renderer=TerratorchRenderer(renderer_config))
 
 
 def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):

vllm/engine/protocol.py

@@ -11,6 +11,7 @@
 from vllm.outputs import PoolingRequestOutput, RequestOutput
 from vllm.plugins.io_processors import IOProcessor
 from vllm.pooling_params import PoolingParams
+from vllm.renderers import RendererLike
 from vllm.sampling_params import SamplingParams
 from vllm.tasks import SupportedTask
 from vllm.tokenizers import TokenizerLike
@@ -27,6 +28,10 @@ class EngineClient(ABC):
     input_processor: InputProcessor
     io_processor: IOProcessor | None
 
+    @property
+    @abstractmethod
+    def renderer(self) -> RendererLike: ...
+
     @property
     @abstractmethod
     def is_running(self) -> bool: ...