[Bugfix][Frontend] Detokenize echoed prompts after truncation (#61876)
Signed-off-by: Cy <15118312+cycancode@users.noreply.github.com> Co-authored-by: Cy <15118312+cycancode@users.noreply.github.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import io
|
import io
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
@@ -376,6 +377,32 @@ class TestRenderPrompt:
|
|||||||
assert results[0]["prompt_token_ids"] == tokens
|
assert results[0]["prompt_token_ids"] == tokens
|
||||||
assert results[0]["prompt"] == "[1, 2, 3, 4]"
|
assert results[0]["prompt"] == "[1, 2, 3, 4]"
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("truncation_side", "expected"),
|
||||||
|
[("left", [106, 107, 108, 109]), ("right", [100, 101, 102, 103])],
|
||||||
|
)
|
||||||
|
def test_detokenization_matches_truncated_tokens(self, truncation_side, expected):
|
||||||
|
"""Echoed prompt text must match the truncated tokens sent to the engine."""
|
||||||
|
renderer = _build_renderer(MockModelConfig())
|
||||||
|
params = TokenizeParams(
|
||||||
|
max_total_tokens=100,
|
||||||
|
truncate_prompt_tokens=4,
|
||||||
|
truncation_side=truncation_side,
|
||||||
|
needs_detokenization=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def render():
|
||||||
|
return renderer.render_prompts(
|
||||||
|
_preprocess_prompt(renderer.model_config, list(range(100, 110)))
|
||||||
|
)
|
||||||
|
|
||||||
|
results = renderer.tokenize_prompts(render(), params)
|
||||||
|
async_results = asyncio.run(renderer.tokenize_prompts_async(render(), params))
|
||||||
|
|
||||||
|
for result in (*results, *async_results):
|
||||||
|
assert result["prompt_token_ids"] == expected
|
||||||
|
assert result["prompt"] == str(expected)
|
||||||
|
|
||||||
def test_explicit_side_tokenizer_unbounded(self):
|
def test_explicit_side_tokenizer_unbounded(self):
|
||||||
renderer = _build_renderer(MockModelConfig())
|
renderer = _build_renderer(MockModelConfig())
|
||||||
|
|
||||||
|
|||||||
+10
-2
@@ -687,13 +687,17 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
prompt = self._tokenize_prompt(prompt, params)
|
prompt = self._tokenize_prompt(prompt, params)
|
||||||
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
|
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
|
||||||
|
|
||||||
|
prompt = params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
# Detokenize after truncation so the text matches the tokens sent
|
||||||
|
# to the engine.
|
||||||
if params.needs_detokenization and "prompt" not in prompt:
|
if params.needs_detokenization and "prompt" not in prompt:
|
||||||
if "prompt_token_ids" not in prompt:
|
if "prompt_token_ids" not in prompt:
|
||||||
raise RuntimeError("Cannot run detokenization on embeddings")
|
raise RuntimeError("Cannot run detokenization on embeddings")
|
||||||
|
|
||||||
prompt = self._detokenize_prompt(prompt) # type: ignore[arg-type]
|
prompt = self._detokenize_prompt(prompt) # type: ignore[arg-type]
|
||||||
|
|
||||||
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
return prompt
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
async def _tokenize_singleton_prompt_async(
|
async def _tokenize_singleton_prompt_async(
|
||||||
@@ -727,13 +731,17 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
prompt = await self._tokenize_prompt_async(prompt, params)
|
prompt = await self._tokenize_prompt_async(prompt, params)
|
||||||
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
|
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
|
||||||
|
|
||||||
|
prompt = params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
# Detokenize after truncation so the text matches the tokens sent
|
||||||
|
# to the engine.
|
||||||
if params.needs_detokenization and "prompt" not in prompt:
|
if params.needs_detokenization and "prompt" not in prompt:
|
||||||
if "prompt_token_ids" not in prompt:
|
if "prompt_token_ids" not in prompt:
|
||||||
raise RuntimeError("Cannot run detokenization on embeddings")
|
raise RuntimeError("Cannot run detokenization on embeddings")
|
||||||
|
|
||||||
prompt = await self._detokenize_prompt_async(prompt) # type: ignore[arg-type]
|
prompt = await self._detokenize_prompt_async(prompt) # type: ignore[arg-type]
|
||||||
|
|
||||||
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
return prompt
|
||||||
|
|
||||||
def _tokenize_enc_dec_prompt(
|
def _tokenize_enc_dec_prompt(
|
||||||
self,
|
self,
|
||||||
|
|||||||
Reference in New Issue
Block a user