diff --git a/tests/renderers/test_token_offsets.py b/tests/renderers/test_token_offsets.py index eca67b142a..fc9192bb43 100644 --- a/tests/renderers/test_token_offsets.py +++ b/tests/renderers/test_token_offsets.py @@ -8,6 +8,8 @@ forwarding chain. Endpoint-level coverage lives in ``tests/entrypoints/scale_out/render/test_render.py``. """ +import asyncio + import pytest from vllm.renderers.params import TokenizeParams @@ -52,6 +54,21 @@ def _make_base_renderer_with(tokenizer): return _StubRenderer(tokenizer) +class _OffsetTokenizer: + """Small deterministic fast tokenizer for pre-tokenization tests.""" + + is_fast = True + max_chars_per_token = 1 + truncation_side = "right" + pad_token_id = 0 + + def __call__(self, text, **kwargs): + return { + "input_ids": list(range(len(text))), + "offset_mapping": [(i, i + 1) for i in range(len(text))], + } + + class TestTokenizePromptOffsets: def test_fast_tokenizer_with_flag_returns_offsets(self, fast_tokenizer): renderer = _make_base_renderer_with(fast_tokenizer) @@ -239,3 +256,44 @@ class TestTruncationKeepsOffsetsAligned: # belonging to the surviving tokens. expected = full_offsets[-keep:] if side == "left" else full_offsets[:keep] assert offsets == expected + + @pytest.mark.parametrize( + ("side", "expected_offsets"), + [ + ("left", [(36, 37), (37, 38), (38, 39), (39, 40)]), + ("right", [(0, 1), (1, 2), (2, 3), (3, 4)]), + ], + ) + def test_text_pretrim_preserves_source_offsets(self, side, expected_offsets): + renderer = _make_base_renderer_with(_OffsetTokenizer()) + text = "0123456789" * 4 + params = TokenizeParams( + max_total_tokens=16, + return_token_offsets=True, + truncate_prompt_tokens=4, + truncation_side=side, + add_special_tokens=False, + ) + + result = renderer.tokenize_prompt({"prompt": text}, params) + + assert result["prompt_token_offsets"] == expected_offsets + + def test_async_left_text_pretrim_preserves_source_offsets(self): + renderer = _make_base_renderer_with(_OffsetTokenizer()) + text = "0123456789" * 4 + params = TokenizeParams( + max_total_tokens=16, + return_token_offsets=True, + truncate_prompt_tokens=4, + truncation_side="left", + ) + + result = asyncio.run(renderer.tokenize_prompt_async({"prompt": text}, params)) + + assert result["prompt_token_offsets"] == [ + (36, 37), + (37, 38), + (38, 39), + (39, 40), + ] diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 9f9f3a30e8..b7eb7c41b5 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -468,6 +468,27 @@ class BaseRenderer(ABC, Generic[_T]): ) return TokensPrompt(prompt_token_ids=list(token_ids), **prompt) + @staticmethod + def _apply_prompt_char_offset( + prompt: TokensPrompt, char_offset: int + ) -> TokensPrompt: + """Map token offsets back to the original prompt after a text pre-trim.""" + if char_offset == 0: + return prompt + + offsets = prompt.get("prompt_token_offsets") + if offsets is not None: + prompt["prompt_token_offsets"] = [ + # Fast tokenizers use (0, 0) for special tokens, which are + # not character spans in the source prompt. + (start, end) + if (start, end) == (0, 0) + else (start + char_offset, end + char_offset) + for start, end in offsets + ] + + return prompt + def _tokenize_prompt( self, prompt: TextPrompt, @@ -523,8 +544,12 @@ class BaseRenderer(ABC, Generic[_T]): "Expected prompt['prompt'] to be a string before tokenization; " "use 'prompt_token_ids' for token ID inputs" ) + prompt_char_offset = params._get_text_truncation_offset( + self.tokenizer, prompt["prompt"] + ) prompt = params.apply_pre_tokenization(self.tokenizer, prompt) # type: ignore[arg-type] prompt = self._tokenize_prompt(prompt, params) + prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset) if params.needs_detokenization and "prompt" not in prompt: if "prompt_token_ids" not in prompt: @@ -559,8 +584,12 @@ class BaseRenderer(ABC, Generic[_T]): "Expected prompt['prompt'] to be a string before tokenization; " "use 'prompt_token_ids' for token ID inputs" ) + prompt_char_offset = params._get_text_truncation_offset( + self.tokenizer, prompt["prompt"] + ) prompt = params.apply_pre_tokenization(self.tokenizer, prompt) # type: ignore[arg-type] prompt = await self._tokenize_prompt_async(prompt, params) + prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset) if params.needs_detokenization and "prompt" not in prompt: if "prompt_token_ids" not in prompt: diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index 856934c4eb..e0eb023dff 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -381,6 +381,31 @@ class TokenizeParams: return text + def _get_text_truncation_offset( + self, tokenizer: TokenizerLike | None, text: str + ) -> int: + """Return the number of source characters removed from the left. + + ``_text_len_check`` pre-truncates long text before tokenization when + an explicit truncation side is requested. Fast-tokenizer offsets are + then relative to that shortened string, so callers need this prefix + length to map them back to the original prompt. + """ + max_input_tokens = self.max_input_tokens + if ( + max_input_tokens is None + or tokenizer is None + or self.truncate_prompt_tokens is None + or self.truncation_side != "left" + ): + return 0 + + max_input_chars = max_input_tokens * tokenizer.max_chars_per_token + if max_input_chars <= 0: + return 0 + + return max(len(text) - max_input_chars, 0) + def _text_lowercase(self, tokenizer: TokenizerLike | None, text: str) -> str: """Apply lowercase to prompt text if necessary.""" return text.lower() if self.do_lower_case else text