[Bugfix][Beam Search] Respect skip_special_tokens during decoding (#56211)
Signed-off-by: lxy-alexander <lxy_alexander@outlook.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
73d2a8cf86
commit
10e8d5614c
@@ -11,8 +11,13 @@ from vllm.sampling_params import BeamSearchParams
|
||||
|
||||
class _Tokenizer:
|
||||
eos_token_id = 0
|
||||
special_token_id = eos_token_id
|
||||
|
||||
def decode(self, token_ids: list[int]) -> str:
|
||||
def decode(self, token_ids: list[int], skip_special_tokens: bool = False) -> str:
|
||||
if skip_special_tokens:
|
||||
token_ids = [
|
||||
token_id for token_id in token_ids if token_id != self.special_token_id
|
||||
]
|
||||
return " ".join(str(token_id) for token_id in token_ids)
|
||||
|
||||
|
||||
@@ -72,3 +77,34 @@ async def test_beam_search_handles_extra_logprob_candidates() -> None:
|
||||
assert outputs[0].outputs[0].finish_reason == "stop"
|
||||
assert outputs[0].outputs[0].token_ids == []
|
||||
assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("skip_special_tokens", "expected_text"),
|
||||
[
|
||||
pytest.param(True, "", id="skip"),
|
||||
pytest.param(False, "0", id="keep"),
|
||||
],
|
||||
)
|
||||
async def test_beam_search_respects_skip_special_tokens(
|
||||
skip_special_tokens: bool, expected_text: str
|
||||
) -> None:
|
||||
prompt = {
|
||||
"type": "token",
|
||||
"prompt": "prompt",
|
||||
"prompt_token_ids": [1],
|
||||
}
|
||||
params = BeamSearchParams(
|
||||
beam_width=1,
|
||||
max_tokens=1,
|
||||
ignore_eos=True,
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
)
|
||||
|
||||
outputs = [
|
||||
output async for output in _Serving().beam_search(prompt, "request", params)
|
||||
]
|
||||
|
||||
assert outputs[0].outputs[0].text == expected_text
|
||||
assert outputs[0].outputs[0].token_ids == [_Tokenizer.special_token_id]
|
||||
|
||||
@@ -186,7 +186,9 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
best_beams = sorted_completed[:beam_width]
|
||||
|
||||
for beam in best_beams:
|
||||
beam.text = tokenizer.decode(beam.tokens)
|
||||
beam.text = tokenizer.decode(
|
||||
beam.tokens, skip_special_tokens=params.skip_special_tokens
|
||||
)
|
||||
|
||||
outputs.append(BeamSearchOutput(sequences=best_beams))
|
||||
|
||||
|
||||
@@ -197,7 +197,9 @@ class BeamSearchOnlineMixin(ABC):
|
||||
tokens = beam.tokens[tokenized_length:-1]
|
||||
else:
|
||||
tokens = beam.tokens[tokenized_length:]
|
||||
beam.text = tokenizer.decode(tokens)
|
||||
beam.text = tokenizer.decode(
|
||||
tokens, skip_special_tokens=params.skip_special_tokens
|
||||
)
|
||||
|
||||
yield RequestOutput(
|
||||
request_id=request_id,
|
||||
|
||||
@@ -659,6 +659,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
temperature=temperature,
|
||||
length_penalty=self.length_penalty,
|
||||
include_stop_str_in_output=self.include_stop_str_in_output,
|
||||
skip_special_tokens=self.skip_special_tokens,
|
||||
)
|
||||
|
||||
def extract_structured_outputs(self) -> StructuredOutputsParams | None:
|
||||
|
||||
@@ -303,6 +303,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
temperature=temperature,
|
||||
length_penalty=self.length_penalty,
|
||||
include_stop_str_in_output=self.include_stop_str_in_output,
|
||||
skip_special_tokens=self.skip_special_tokens,
|
||||
)
|
||||
|
||||
def extract_structured_outputs(self) -> StructuredOutputsParams | None:
|
||||
|
||||
@@ -1313,6 +1313,7 @@ class BeamSearchParams(
|
||||
length_penalty: float = 1.0
|
||||
include_stop_str_in_output: bool = False
|
||||
structured_outputs: StructuredOutputsParams | None = None
|
||||
skip_special_tokens: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_verify_num_sequences(self.beam_width, "beam_width")
|
||||
|
||||
Reference in New Issue
Block a user