From 10e8d5614c50ad180acbe61878c93d98ca7098a9 Mon Sep 17 00:00:00 2001 From: Alexander Lee Date: Mon, 14 Sep 2026 01:49:44 -0400 Subject: [PATCH] [Bugfix][Beam Search] Respect skip_special_tokens during decoding (#56211) Signed-off-by: lxy-alexander Co-authored-by: OpenAI Codex --- tests/samplers/test_beam_search_online.py | 38 ++++++++++++++++++- .../generate/beam_search/offline.py | 4 +- .../generate/beam_search/online.py | 4 +- .../openai/chat_completion/protocol.py | 1 + .../entrypoints/openai/completion/protocol.py | 1 + vllm/sampling_params.py | 1 + 6 files changed, 46 insertions(+), 3 deletions(-) diff --git a/tests/samplers/test_beam_search_online.py b/tests/samplers/test_beam_search_online.py index 14481f79c6..102e66dcbe 100644 --- a/tests/samplers/test_beam_search_online.py +++ b/tests/samplers/test_beam_search_online.py @@ -11,8 +11,13 @@ from vllm.sampling_params import BeamSearchParams class _Tokenizer: eos_token_id = 0 + special_token_id = eos_token_id - def decode(self, token_ids: list[int]) -> str: + def decode(self, token_ids: list[int], skip_special_tokens: bool = False) -> str: + if skip_special_tokens: + token_ids = [ + token_id for token_id in token_ids if token_id != self.special_token_id + ] return " ".join(str(token_id) for token_id in token_ids) @@ -72,3 +77,34 @@ async def test_beam_search_handles_extra_logprob_candidates() -> None: assert outputs[0].outputs[0].finish_reason == "stop" assert outputs[0].outputs[0].token_ids == [] assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("skip_special_tokens", "expected_text"), + [ + pytest.param(True, "", id="skip"), + pytest.param(False, "0", id="keep"), + ], +) +async def test_beam_search_respects_skip_special_tokens( + skip_special_tokens: bool, expected_text: str +) -> None: + prompt = { + "type": "token", + "prompt": "prompt", + "prompt_token_ids": [1], + } + params = BeamSearchParams( + beam_width=1, + max_tokens=1, + ignore_eos=True, + skip_special_tokens=skip_special_tokens, + ) + + outputs = [ + output async for output in _Serving().beam_search(prompt, "request", params) + ] + + assert outputs[0].outputs[0].text == expected_text + assert outputs[0].outputs[0].token_ids == [_Tokenizer.special_token_id] diff --git a/vllm/entrypoints/generate/beam_search/offline.py b/vllm/entrypoints/generate/beam_search/offline.py index f412f5aebe..b919435f65 100644 --- a/vllm/entrypoints/generate/beam_search/offline.py +++ b/vllm/entrypoints/generate/beam_search/offline.py @@ -186,7 +186,9 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin): best_beams = sorted_completed[:beam_width] for beam in best_beams: - beam.text = tokenizer.decode(beam.tokens) + beam.text = tokenizer.decode( + beam.tokens, skip_special_tokens=params.skip_special_tokens + ) outputs.append(BeamSearchOutput(sequences=best_beams)) diff --git a/vllm/entrypoints/generate/beam_search/online.py b/vllm/entrypoints/generate/beam_search/online.py index 02ff61d629..ac1a585e8f 100644 --- a/vllm/entrypoints/generate/beam_search/online.py +++ b/vllm/entrypoints/generate/beam_search/online.py @@ -197,7 +197,9 @@ class BeamSearchOnlineMixin(ABC): tokens = beam.tokens[tokenized_length:-1] else: tokens = beam.tokens[tokenized_length:] - beam.text = tokenizer.decode(tokens) + beam.text = tokenizer.decode( + tokens, skip_special_tokens=params.skip_special_tokens + ) yield RequestOutput( request_id=request_id, diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index b7b292e835..badd5a0344 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -659,6 +659,7 @@ class ChatCompletionRequest(OpenAIBaseModel): temperature=temperature, length_penalty=self.length_penalty, include_stop_str_in_output=self.include_stop_str_in_output, + skip_special_tokens=self.skip_special_tokens, ) def extract_structured_outputs(self) -> StructuredOutputsParams | None: diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index 6d7a49bacd..bf0a94eea7 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -303,6 +303,7 @@ class CompletionRequest(OpenAIBaseModel): temperature=temperature, length_penalty=self.length_penalty, include_stop_str_in_output=self.include_stop_str_in_output, + skip_special_tokens=self.skip_special_tokens, ) def extract_structured_outputs(self) -> StructuredOutputsParams | None: diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 52a65aefbc..c0ab45ced7 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -1313,6 +1313,7 @@ class BeamSearchParams( length_penalty: float = 1.0 include_stop_str_in_output: bool = False structured_outputs: StructuredOutputsParams | None = None + skip_special_tokens: bool = True def __post_init__(self) -> None: _verify_num_sequences(self.beam_width, "beam_width")