[Bugfix][Beam Search] Respect skip_special_tokens during decoding (#56211)

Signed-off-by: lxy-alexander <lxy_alexander@outlook.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Alexander Lee
2026-09-14 05:49:44 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 73d2a8cf86
commit 10e8d5614c
6 changed files with 46 additions and 3 deletions
+37 -1
View File
@@ -11,8 +11,13 @@ from vllm.sampling_params import BeamSearchParams
class _Tokenizer:
eos_token_id = 0
special_token_id = eos_token_id
def decode(self, token_ids: list[int]) -> str:
def decode(self, token_ids: list[int], skip_special_tokens: bool = False) -> str:
if skip_special_tokens:
token_ids = [
token_id for token_id in token_ids if token_id != self.special_token_id
]
return " ".join(str(token_id) for token_id in token_ids)
@@ -72,3 +77,34 @@ async def test_beam_search_handles_extra_logprob_candidates() -> None:
assert outputs[0].outputs[0].finish_reason == "stop"
assert outputs[0].outputs[0].token_ids == []
assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("skip_special_tokens", "expected_text"),
[
pytest.param(True, "", id="skip"),
pytest.param(False, "0", id="keep"),
],
)
async def test_beam_search_respects_skip_special_tokens(
skip_special_tokens: bool, expected_text: str
) -> None:
prompt = {
"type": "token",
"prompt": "prompt",
"prompt_token_ids": [1],
}
params = BeamSearchParams(
beam_width=1,
max_tokens=1,
ignore_eos=True,
skip_special_tokens=skip_special_tokens,
)
outputs = [
output async for output in _Serving().beam_search(prompt, "request", params)
]
assert outputs[0].outputs[0].text == expected_text
assert outputs[0].outputs[0].token_ids == [_Tokenizer.special_token_id]
@@ -186,7 +186,9 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
best_beams = sorted_completed[:beam_width]
for beam in best_beams:
beam.text = tokenizer.decode(beam.tokens)
beam.text = tokenizer.decode(
beam.tokens, skip_special_tokens=params.skip_special_tokens
)
outputs.append(BeamSearchOutput(sequences=best_beams))
@@ -197,7 +197,9 @@ class BeamSearchOnlineMixin(ABC):
tokens = beam.tokens[tokenized_length:-1]
else:
tokens = beam.tokens[tokenized_length:]
beam.text = tokenizer.decode(tokens)
beam.text = tokenizer.decode(
tokens, skip_special_tokens=params.skip_special_tokens
)
yield RequestOutput(
request_id=request_id,
@@ -659,6 +659,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
temperature=temperature,
length_penalty=self.length_penalty,
include_stop_str_in_output=self.include_stop_str_in_output,
skip_special_tokens=self.skip_special_tokens,
)
def extract_structured_outputs(self) -> StructuredOutputsParams | None:
@@ -303,6 +303,7 @@ class CompletionRequest(OpenAIBaseModel):
temperature=temperature,
length_penalty=self.length_penalty,
include_stop_str_in_output=self.include_stop_str_in_output,
skip_special_tokens=self.skip_special_tokens,
)
def extract_structured_outputs(self) -> StructuredOutputsParams | None:
+1
View File
@@ -1313,6 +1313,7 @@ class BeamSearchParams(
length_penalty: float = 1.0
include_stop_str_in_output: bool = False
structured_outputs: StructuredOutputsParams | None = None
skip_special_tokens: bool = True
def __post_init__(self) -> None:
_verify_num_sequences(self.beam_width, "beam_width")