From d74f4180cdff64d410084173d7f9bf73af70f615 Mon Sep 17 00:00:00 2001 From: Cy Cheng Date: Fri, 16 Oct 2026 22:20:38 -0700 Subject: [PATCH] [Bugfix][Frontend] Detokenize echoed prompts after truncation (#61876) Signed-off-by: Cy <15118312+cycancode@users.noreply.github.com> Co-authored-by: Cy <15118312+cycancode@users.noreply.github.com> --- tests/renderers/test_completions.py | 27 +++++++++++++++++++++++++++ vllm/renderers/base.py | 12 ++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/renderers/test_completions.py b/tests/renderers/test_completions.py index 9d37ff78d1..f7e53b1389 100644 --- a/tests/renderers/test_completions.py +++ b/tests/renderers/test_completions.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio import io from collections.abc import Sequence from dataclasses import dataclass @@ -376,6 +377,32 @@ class TestRenderPrompt: assert results[0]["prompt_token_ids"] == tokens assert results[0]["prompt"] == "[1, 2, 3, 4]" + @pytest.mark.parametrize( + ("truncation_side", "expected"), + [("left", [106, 107, 108, 109]), ("right", [100, 101, 102, 103])], + ) + def test_detokenization_matches_truncated_tokens(self, truncation_side, expected): + """Echoed prompt text must match the truncated tokens sent to the engine.""" + renderer = _build_renderer(MockModelConfig()) + params = TokenizeParams( + max_total_tokens=100, + truncate_prompt_tokens=4, + truncation_side=truncation_side, + needs_detokenization=True, + ) + + def render(): + return renderer.render_prompts( + _preprocess_prompt(renderer.model_config, list(range(100, 110))) + ) + + results = renderer.tokenize_prompts(render(), params) + async_results = asyncio.run(renderer.tokenize_prompts_async(render(), params)) + + for result in (*results, *async_results): + assert result["prompt_token_ids"] == expected + assert result["prompt"] == str(expected) + def test_explicit_side_tokenizer_unbounded(self): renderer = _build_renderer(MockModelConfig()) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 7b0f4debaa..f22b4215bd 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -687,13 +687,17 @@ class BaseRenderer(ABC, Generic[_T]): prompt = self._tokenize_prompt(prompt, params) prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset) + prompt = params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type] + + # Detokenize after truncation so the text matches the tokens sent + # to the engine. if params.needs_detokenization and "prompt" not in prompt: if "prompt_token_ids" not in prompt: raise RuntimeError("Cannot run detokenization on embeddings") prompt = self._detokenize_prompt(prompt) # type: ignore[arg-type] - return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type] + return prompt @overload async def _tokenize_singleton_prompt_async( @@ -727,13 +731,17 @@ class BaseRenderer(ABC, Generic[_T]): prompt = await self._tokenize_prompt_async(prompt, params) prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset) + prompt = params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type] + + # Detokenize after truncation so the text matches the tokens sent + # to the engine. if params.needs_detokenization and "prompt" not in prompt: if "prompt_token_ids" not in prompt: raise RuntimeError("Cannot run detokenization on embeddings") prompt = await self._detokenize_prompt_async(prompt) # type: ignore[arg-type] - return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type] + return prompt def _tokenize_enc_dec_prompt( self,