From b7b20b1b8f8014e216da573351d2652b3963ada6 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 15 Oct 2026 06:51:34 -0700 Subject: [PATCH] [Bugfix][Frontend] Leave the final stop token out of chat derender text (#61757) Signed-off-by: khluu Co-authored-by: Claude Opus 5.5 Co-authored-by: Cyrus Leung --- .../derender/test_derender_stream.py | 79 +++++++++++++++++++ vllm/renderers/online_derenderer.py | 56 ++++++++++++- 2 files changed, 132 insertions(+), 3 deletions(-) diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index a1c4e2a330..372ef31ea3 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -348,6 +348,85 @@ def test_non_streaming_derender_initializes_parser_prefix( assert choices[0].finish_reason == "length" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("stop", "include_stop_str_in_output", "dropped"), + [ + ("eos", False, True), + ("stop_token_id", False, True), + ("stop_string", False, False), + ("eos", True, False), + ], +) +async def test_parsed_derender_leaves_out_stop_token_like_engine( + tokenizer, stop, include_stop_str_in_output, dropped +): + """A parser keeps special tokens in the decoded text, so the token that + stopped generation must be dropped the way the engine drops it.""" + eos = tokenizer.eos_token_id + answer_ids = tokenizer.encode("answer", add_special_tokens=False) + bang = tokenizer.encode("!", add_special_tokens=False)[-1] + last_id, stop_reason = { + "eos": (eos, None), + "stop_token_id": (bang, bang), + "stop_string": (bang, "!"), + }[stop] + + parser = MagicMock() + parser.parse.side_effect = lambda text, *args, **kwargs: (None, text, []) + parser.parse_delta.side_effect = lambda text, *args, **kwargs: DeltaMessage( + content=text + ) + request = _chat_request(include_stop_str_in_output=include_stop_str_in_output) + + with ThreadPoolExecutor(max_workers=1) as executor: + renderer = MagicMock() + renderer.get_tokenizer.return_value = tokenizer + renderer.get_eos_token_id.return_value = eos + renderer._executor = executor + model_config = MagicMock() + model_config.hf_config.model_type = "llama" + model_config.model = MODEL_NAME + model_config.try_get_generation_config.return_value = {} + derenderer = OnlineDerenderer( + model_config=model_config, + renderer=renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="string", + ) + derenderer.parser = MagicMock(return_value=parser) + + def batch_content(token_ids: list[int], finish_reason: str) -> str: + choice = GenerateTokensChoice( + index=0, + token_ids=token_ids, + finish_reason=finish_reason, + stop_reason=stop_reason if finish_reason == "stop" else None, + ) + response = GenerateTokensResponse( + request_id="test", prompt_token_ids=[11, 12], choices=[choice] + ) + return derenderer._derender_chat(response, request)[0].message.content + + # Same decode window and seeding; "length" never drops a token. + expected = batch_content( + answer_ids if dropped else answer_ids + [last_id], "length" + ) + batch = batch_content(answer_ids + [last_id], "stop") + stream, _ = await derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + answer_ids + [last_id], finish_reason="stop", stop_reason=stop_reason + ), + chat_request=request, + prompt_token_ids=[11, 12], + ) + + assert batch == expected + assert stream.choices[0].delta.content == expected + + class TestDetokenizeDelta: """_detokenize_delta: chunked decode must equal one shot decode.""" diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index 09ea33857c..9a22df962e 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Sequence +from functools import cached_property from typing import Any from vllm.config import ModelConfig @@ -130,6 +131,43 @@ class OnlineDerenderer: generate_response, chat_request, prompt_token_ids ) + @cached_property + def _model_eos_token_ids(self) -> frozenset[int]: + """EOS ids the engine stops on: the tokenizer's and generation_config's.""" + eos_ids: set[int] = set() + if (eos_token_id := self.renderer.get_eos_token_id()) is not None: + eos_ids.add(eos_token_id) + config_eos = self.model_config.try_get_generation_config().get("eos_token_id") + if config_eos is not None: + eos_ids.update([config_eos] if isinstance(config_eos, int) else config_eos) + return frozenset(eos_ids) + + def _text_token_ids( + self, + token_ids: list[int], + finish_reason: str | None, + stop_reason: int | str | None, + request: ChatCompletionRequest | None, + ) -> list[int]: + """The ids whose text the engine returns. Like its detokenizer, leave + out the token that stopped generation (EOS when `stop_reason` is None, + a `stop_token_ids` entry when it is an int) unless + `include_stop_str_in_output` is set. The last-id check guards against + `finish_reason` defaulting to "stop" when the client sends neither.""" + if ( + finish_reason != "stop" + or isinstance(stop_reason, str) + or not token_ids + or (request is not None and request.include_stop_str_in_output) + ): + return token_ids + last = token_ids[-1] + if stop_reason is None: + stopped = last in self._model_eos_token_ids + else: + stopped = last == stop_reason + return token_ids[:-1] if stopped else token_ids + def _derender_chat( self, generate_response: GenerateTokensResponse, @@ -182,7 +220,12 @@ class OnlineDerenderer: # (default True when no request was given). decoded_text, _ = self._detokenize_delta( tokenizer, - choice.token_ids, + self._text_token_ids( + choice.token_ids, + choice.finish_reason, + choice.stop_reason, + chat_request, + ), seed_state, skip_special_tokens=skip_special, spaces_between_special_tokens=spaces_between, @@ -447,7 +490,9 @@ class OnlineDerenderer: delta_tids = choice.token_ids or [] new_text, updated_state = await self._detokenize_delta_async( tokenizer, - delta_tids, + self._text_token_ids( + delta_tids, choice.finish_reason, choice.stop_reason, chat_request + ), updated_state, skip_special_tokens=skip_special, spaces_between_special_tokens=spaces_between, @@ -617,7 +662,12 @@ class OnlineDerenderer: # in this method's docstring. text, detok_state = self._detokenize_delta( tokenizer, - delta_tids, + self._text_token_ids( + delta_tids, + choice.finish_reason, + choice.stop_reason, + chat_request, + ), detok_state, skip_special_tokens=skip_special, spaces_between_special_tokens=spaces_between,