[Bugfix][Frontend] Preserve token offset origins after left text pre-trimming (#54692)
Signed-off-by: waizuichougou <2082431897@qq.com>
This commit is contained in:
@@ -8,6 +8,8 @@ forwarding chain. Endpoint-level coverage lives in
|
||||
``tests/entrypoints/scale_out/render/test_render.py``.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.renderers.params import TokenizeParams
|
||||
@@ -52,6 +54,21 @@ def _make_base_renderer_with(tokenizer):
|
||||
return _StubRenderer(tokenizer)
|
||||
|
||||
|
||||
class _OffsetTokenizer:
|
||||
"""Small deterministic fast tokenizer for pre-tokenization tests."""
|
||||
|
||||
is_fast = True
|
||||
max_chars_per_token = 1
|
||||
truncation_side = "right"
|
||||
pad_token_id = 0
|
||||
|
||||
def __call__(self, text, **kwargs):
|
||||
return {
|
||||
"input_ids": list(range(len(text))),
|
||||
"offset_mapping": [(i, i + 1) for i in range(len(text))],
|
||||
}
|
||||
|
||||
|
||||
class TestTokenizePromptOffsets:
|
||||
def test_fast_tokenizer_with_flag_returns_offsets(self, fast_tokenizer):
|
||||
renderer = _make_base_renderer_with(fast_tokenizer)
|
||||
@@ -239,3 +256,44 @@ class TestTruncationKeepsOffsetsAligned:
|
||||
# belonging to the surviving tokens.
|
||||
expected = full_offsets[-keep:] if side == "left" else full_offsets[:keep]
|
||||
assert offsets == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("side", "expected_offsets"),
|
||||
[
|
||||
("left", [(36, 37), (37, 38), (38, 39), (39, 40)]),
|
||||
("right", [(0, 1), (1, 2), (2, 3), (3, 4)]),
|
||||
],
|
||||
)
|
||||
def test_text_pretrim_preserves_source_offsets(self, side, expected_offsets):
|
||||
renderer = _make_base_renderer_with(_OffsetTokenizer())
|
||||
text = "0123456789" * 4
|
||||
params = TokenizeParams(
|
||||
max_total_tokens=16,
|
||||
return_token_offsets=True,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side=side,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
|
||||
result = renderer.tokenize_prompt({"prompt": text}, params)
|
||||
|
||||
assert result["prompt_token_offsets"] == expected_offsets
|
||||
|
||||
def test_async_left_text_pretrim_preserves_source_offsets(self):
|
||||
renderer = _make_base_renderer_with(_OffsetTokenizer())
|
||||
text = "0123456789" * 4
|
||||
params = TokenizeParams(
|
||||
max_total_tokens=16,
|
||||
return_token_offsets=True,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side="left",
|
||||
)
|
||||
|
||||
result = asyncio.run(renderer.tokenize_prompt_async({"prompt": text}, params))
|
||||
|
||||
assert result["prompt_token_offsets"] == [
|
||||
(36, 37),
|
||||
(37, 38),
|
||||
(38, 39),
|
||||
(39, 40),
|
||||
]
|
||||
|
||||
@@ -468,6 +468,27 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
)
|
||||
return TokensPrompt(prompt_token_ids=list(token_ids), **prompt)
|
||||
|
||||
@staticmethod
|
||||
def _apply_prompt_char_offset(
|
||||
prompt: TokensPrompt, char_offset: int
|
||||
) -> TokensPrompt:
|
||||
"""Map token offsets back to the original prompt after a text pre-trim."""
|
||||
if char_offset == 0:
|
||||
return prompt
|
||||
|
||||
offsets = prompt.get("prompt_token_offsets")
|
||||
if offsets is not None:
|
||||
prompt["prompt_token_offsets"] = [
|
||||
# Fast tokenizers use (0, 0) for special tokens, which are
|
||||
# not character spans in the source prompt.
|
||||
(start, end)
|
||||
if (start, end) == (0, 0)
|
||||
else (start + char_offset, end + char_offset)
|
||||
for start, end in offsets
|
||||
]
|
||||
|
||||
return prompt
|
||||
|
||||
def _tokenize_prompt(
|
||||
self,
|
||||
prompt: TextPrompt,
|
||||
@@ -523,8 +544,12 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
"Expected prompt['prompt'] to be a string before tokenization; "
|
||||
"use 'prompt_token_ids' for token ID inputs"
|
||||
)
|
||||
prompt_char_offset = params._get_text_truncation_offset(
|
||||
self.tokenizer, prompt["prompt"]
|
||||
)
|
||||
prompt = params.apply_pre_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
prompt = self._tokenize_prompt(prompt, params)
|
||||
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
|
||||
|
||||
if params.needs_detokenization and "prompt" not in prompt:
|
||||
if "prompt_token_ids" not in prompt:
|
||||
@@ -559,8 +584,12 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
"Expected prompt['prompt'] to be a string before tokenization; "
|
||||
"use 'prompt_token_ids' for token ID inputs"
|
||||
)
|
||||
prompt_char_offset = params._get_text_truncation_offset(
|
||||
self.tokenizer, prompt["prompt"]
|
||||
)
|
||||
prompt = params.apply_pre_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
prompt = await self._tokenize_prompt_async(prompt, params)
|
||||
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
|
||||
|
||||
if params.needs_detokenization and "prompt" not in prompt:
|
||||
if "prompt_token_ids" not in prompt:
|
||||
|
||||
@@ -381,6 +381,31 @@ class TokenizeParams:
|
||||
|
||||
return text
|
||||
|
||||
def _get_text_truncation_offset(
|
||||
self, tokenizer: TokenizerLike | None, text: str
|
||||
) -> int:
|
||||
"""Return the number of source characters removed from the left.
|
||||
|
||||
``_text_len_check`` pre-truncates long text before tokenization when
|
||||
an explicit truncation side is requested. Fast-tokenizer offsets are
|
||||
then relative to that shortened string, so callers need this prefix
|
||||
length to map them back to the original prompt.
|
||||
"""
|
||||
max_input_tokens = self.max_input_tokens
|
||||
if (
|
||||
max_input_tokens is None
|
||||
or tokenizer is None
|
||||
or self.truncate_prompt_tokens is None
|
||||
or self.truncation_side != "left"
|
||||
):
|
||||
return 0
|
||||
|
||||
max_input_chars = max_input_tokens * tokenizer.max_chars_per_token
|
||||
if max_input_chars <= 0:
|
||||
return 0
|
||||
|
||||
return max(len(text) - max_input_chars, 0)
|
||||
|
||||
def _text_lowercase(self, tokenizer: TokenizerLike | None, text: str) -> str:
|
||||
"""Apply lowercase to prompt text if necessary."""
|
||||
return text.lower() if self.do_lower_case else text
|
||||
|
||||
Reference in New Issue
Block a user