[Bugfix][Frontend] Leave the final stop token out of chat derender text (#61757)

Signed-off-by: khluu <khluu000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

tests/entrypoints/scale_out/derender/test_derender_stream.py

@@ -348,6 +348,85 @@ def test_non_streaming_derender_initializes_parser_prefix(
     assert choices[0].finish_reason == "length"
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("stop", "include_stop_str_in_output", "dropped"),
+    [
+        ("eos", False, True),
+        ("stop_token_id", False, True),
+        ("stop_string", False, False),
+        ("eos", True, False),
+    ],
+)
+async def test_parsed_derender_leaves_out_stop_token_like_engine(
+    tokenizer, stop, include_stop_str_in_output, dropped
+):
+    """A parser keeps special tokens in the decoded text, so the token that
+    stopped generation must be dropped the way the engine drops it."""
+    eos = tokenizer.eos_token_id
+    answer_ids = tokenizer.encode("answer", add_special_tokens=False)
+    bang = tokenizer.encode("!", add_special_tokens=False)[-1]
+    last_id, stop_reason = {
+        "eos": (eos, None),
+        "stop_token_id": (bang, bang),
+        "stop_string": (bang, "!"),
+    }[stop]
+
+    parser = MagicMock()
+    parser.parse.side_effect = lambda text, *args, **kwargs: (None, text, [])
+    parser.parse_delta.side_effect = lambda text, *args, **kwargs: DeltaMessage(
+        content=text
+    )
+    request = _chat_request(include_stop_str_in_output=include_stop_str_in_output)
+
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        renderer = MagicMock()
+        renderer.get_tokenizer.return_value = tokenizer
+        renderer.get_eos_token_id.return_value = eos
+        renderer._executor = executor
+        model_config = MagicMock()
+        model_config.hf_config.model_type = "llama"
+        model_config.model = MODEL_NAME
+        model_config.try_get_generation_config.return_value = {}
+        derenderer = OnlineDerenderer(
+            model_config=model_config,
+            renderer=renderer,
+            request_logger=None,
+            chat_template=None,
+            chat_template_content_format="string",
+        )
+        derenderer.parser = MagicMock(return_value=parser)
+
+        def batch_content(token_ids: list[int], finish_reason: str) -> str:
+            choice = GenerateTokensChoice(
+                index=0,
+                token_ids=token_ids,
+                finish_reason=finish_reason,
+                stop_reason=stop_reason if finish_reason == "stop" else None,
+            )
+            response = GenerateTokensResponse(
+                request_id="test", prompt_token_ids=[11, 12], choices=[choice]
+            )
+            return derenderer._derender_chat(response, request)[0].message.content
+
+        # Same decode window and seeding; "length" never drops a token.
+        expected = batch_content(
+            answer_ids if dropped else answer_ids + [last_id], "length"
+        )
+        batch = batch_content(answer_ids + [last_id], "stop")
+        stream, _ = await derenderer.derender_chat_stream(
+            model=MODEL_NAME,
+            generate_chunk=_make_stream_chunk(
+                answer_ids + [last_id], finish_reason="stop", stop_reason=stop_reason
+            ),
+            chat_request=request,
+            prompt_token_ids=[11, 12],
+        )
+
+    assert batch == expected
+    assert stream.choices[0].delta.content == expected
+
+
 class TestDetokenizeDelta:
     """_detokenize_delta: chunked decode must equal one shot decode."""
 

vllm/renderers/online_derenderer.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 from collections.abc import Sequence
+from functools import cached_property
 from typing import Any
 
 from vllm.config import ModelConfig
@@ -130,6 +131,43 @@ class OnlineDerenderer:
             generate_response, chat_request, prompt_token_ids
         )
 
+    @cached_property
+    def _model_eos_token_ids(self) -> frozenset[int]:
+        """EOS ids the engine stops on: the tokenizer's and generation_config's."""
+        eos_ids: set[int] = set()
+        if (eos_token_id := self.renderer.get_eos_token_id()) is not None:
+            eos_ids.add(eos_token_id)
+        config_eos = self.model_config.try_get_generation_config().get("eos_token_id")
+        if config_eos is not None:
+            eos_ids.update([config_eos] if isinstance(config_eos, int) else config_eos)
+        return frozenset(eos_ids)
+
+    def _text_token_ids(
+        self,
+        token_ids: list[int],
+        finish_reason: str | None,
+        stop_reason: int | str | None,
+        request: ChatCompletionRequest | None,
+    ) -> list[int]:
+        """The ids whose text the engine returns. Like its detokenizer, leave
+        out the token that stopped generation (EOS when `stop_reason` is None,
+        a `stop_token_ids` entry when it is an int) unless
+        `include_stop_str_in_output` is set. The last-id check guards against
+        `finish_reason` defaulting to "stop" when the client sends neither."""
+        if (
+            finish_reason != "stop"
+            or isinstance(stop_reason, str)
+            or not token_ids
+            or (request is not None and request.include_stop_str_in_output)
+        ):
+            return token_ids
+        last = token_ids[-1]
+        if stop_reason is None:
+            stopped = last in self._model_eos_token_ids
+        else:
+            stopped = last == stop_reason
+        return token_ids[:-1] if stopped else token_ids
+
     def _derender_chat(
         self,
         generate_response: GenerateTokensResponse,
@@ -182,7 +220,12 @@ class OnlineDerenderer:
             # (default True when no request was given).
             decoded_text, _ = self._detokenize_delta(
                 tokenizer,
-                choice.token_ids,
+                self._text_token_ids(
+                    choice.token_ids,
+                    choice.finish_reason,
+                    choice.stop_reason,
+                    chat_request,
+                ),
                 seed_state,
                 skip_special_tokens=skip_special,
                 spaces_between_special_tokens=spaces_between,
@@ -447,7 +490,9 @@ class OnlineDerenderer:
             delta_tids = choice.token_ids or []
             new_text, updated_state = await self._detokenize_delta_async(
                 tokenizer,
-                delta_tids,
+                self._text_token_ids(
+                    delta_tids, choice.finish_reason, choice.stop_reason, chat_request
+                ),
                 updated_state,
                 skip_special_tokens=skip_special,
                 spaces_between_special_tokens=spaces_between,
@@ -617,7 +662,12 @@ class OnlineDerenderer:
                 # in this method's docstring.
                 text, detok_state = self._detokenize_delta(
                     tokenizer,
-                    delta_tids,
+                    self._text_token_ids(
+                        delta_tids,
+                        choice.finish_reason,
+                        choice.stop_reason,
+                        chat_request,
+                    ),
                     detok_state,
                     skip_special_tokens=skip_special,
                     spaces_between_special_tokens=spaces_between,