[Renderer] Move InputPreprocessor into Renderer (1/2) (#34510)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
Cyrus Leung
2026-02-14 10:14:21 -08:00
committed by GitHub
parent b3c14229b0
commit 73391a1baa
39 changed files with 456 additions and 458 deletions
+12 -10
View File
@@ -38,6 +38,7 @@ class MockModelConfig:
enable_prompt_embeds: bool = True
skip_tokenizer_init: bool = False
is_encoder_decoder: bool = False
is_multimodal_model: bool = False
@dataclass
@@ -78,15 +79,16 @@ def _build_renderer(
renderer = HfRenderer(
MockVllmConfig(model_config),
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
tokenizer=(
None
if model_config.skip_tokenizer_init
else DummyTokenizer(
truncation_side=truncation_side,
max_chars_per_token=max_chars_per_token,
)
),
)
if not model_config.skip_tokenizer_init:
renderer._tokenizer = DummyTokenizer(
truncation_side=truncation_side,
max_chars_per_token=max_chars_per_token,
)
return renderer
@@ -277,7 +279,7 @@ class TestRenderPrompt:
)
# Should not even attempt tokenization
assert renderer._tokenizer._captured_encode_kwargs == {}
assert renderer.tokenizer._captured_encode_kwargs == {}
def test_text_max_length_exceeded_nonobvious(self):
renderer = _build_renderer(MockModelConfig(), max_chars_per_token=2)
@@ -298,8 +300,8 @@ class TestRenderPrompt:
)
# Should only tokenize the first max_total_tokens + 1 tokens
assert renderer._tokenizer._captured_encode_kwargs["truncation"] is True
assert renderer._tokenizer._captured_encode_kwargs["max_length"] == 101
assert renderer.tokenizer._captured_encode_kwargs["truncation"] is True
assert renderer.tokenizer._captured_encode_kwargs["max_length"] == 101
def test_token_max_length_exceeded(self):
renderer = _build_renderer(MockModelConfig())
+2 -2
View File
@@ -36,6 +36,7 @@ class MockModelConfig:
enable_prompt_embeds: bool = True
skip_tokenizer_init: bool = False
is_encoder_decoder: bool = False
is_multimodal_model: bool = False
@dataclass
@@ -57,9 +58,8 @@ async def test_async_mistral_tokenizer_does_not_block_event_loop():
mock_tokenizer.apply_chat_template = mocked_apply_chat_template
mock_renderer = MistralRenderer(
MockVllmConfig(mock_model_config),
tokenizer_kwargs={},
tokenizer=mock_tokenizer,
)
mock_renderer._tokenizer = mock_tokenizer
task = mock_renderer.render_messages_async([], ChatParams())