[Bugfix][Frontend] Stop beam search on generation config EOS tokens (#62072)
pre-commit / pre-run-check (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s

Signed-off-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
Co-authored-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
This commit is contained in:
alp
2026-10-18 16:49:42 +00:00
committed by GitHub
co-authored by mottopanikeiku
parent fe8f2140ee
commit ff01a6b14f
5 changed files with 110 additions and 16 deletions
@@ -27,6 +27,7 @@ from .utils import (
BeamSearchOutput,
BeamSearchSequence,
create_sort_beams_key_function,
get_beam_search_eos_token_ids,
)
logger = init_logger(__name__)
@@ -86,8 +87,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
self.llm_engine.input_processor.resolve_watermarking(params)
tokenizer = self.renderer.get_tokenizer()
eos_token_id = tokenizer.eos_token_id
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
engine_inputs = self._preprocess_cmpl(prompts)
lora_requests = self._lora_request_to_seq(lora_request, len(engine_inputs))
@@ -129,6 +128,11 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
params.structured_outputs, tokenizer
)
eos_token_ids = get_beam_search_eos_token_ids(
self.llm_engine.input_processor.generation_config_fields,
tokenizer.eos_token_id,
)
sort_beams_key = create_sort_beams_key_function(eos_token_ids, length_penalty)
instances: list[BeamSearchInstance] = []
for lora_req, prompt in zip(lora_requests, engine_inputs):
@@ -169,7 +173,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
should_stop = self._beam_search_step(
instances_batch=instances_batch,
base_sampling_params=base_sampling_params,
eos_token_id=eos_token_id,
eos_token_ids=eos_token_ids,
ignore_eos=ignore_eos,
beam_width=beam_width,
sort_beams_key=sort_beams_key,
@@ -204,7 +208,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
self,
instances_batch: list[BeamSearchInstance],
base_sampling_params: SamplingParams,
eos_token_id: int | None,
eos_token_ids: set[int],
ignore_eos: bool,
beam_width: int,
sort_beams_key: Callable,
@@ -350,7 +354,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
cum_logprob=current_beam.cum_logprob + logprob_obj.logprob,
)
if token_id == eos_token_id and not ignore_eos:
if token_id in eos_token_ids and not ignore_eos:
instance.completed.append(new_beam)
else:
instance_new_beams.append(new_beam)
@@ -20,6 +20,7 @@ from vllm.utils.async_utils import collect_from_async_generator
from .utils import (
BeamSearchSequence,
create_sort_beams_key_function,
get_beam_search_eos_token_ids,
)
@@ -47,8 +48,6 @@ class BeamSearchOnlineMixin(ABC):
self.engine_client.input_processor.resolve_watermarking(params)
tokenizer = self.renderer.get_tokenizer()
eos_token_id = tokenizer.eos_token_id
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
if prompt["type"] == "embeds":
raise NotImplementedError("Embedding prompt not supported for beam search")
@@ -74,6 +73,11 @@ class BeamSearchOnlineMixin(ABC):
sampling_params, self.engine_client.input_processor.vllm_config
)
allowed = set(params.allowed_token_ids) if params.allowed_token_ids else None
eos_token_ids = get_beam_search_eos_token_ids(
self.engine_client.input_processor.generation_config_fields,
tokenizer.eos_token_id,
)
sort_beams_key = create_sort_beams_key_function(eos_token_ids, length_penalty)
all_beams = [
BeamSearchSequence(
orig_prompt=prompt,
@@ -159,17 +163,17 @@ class BeamSearchOnlineMixin(ABC):
candidate_logprob = (
current_beam.cum_logprob + logprob_obj.logprob
)
if token_id == eos_token_id and not ignore_eos:
if token_id in eos_token_ids and not ignore_eos:
completed.append(
BeamSearchSequence(
orig_prompt=prompt,
tokens=current_beam.tokens + [eos_token_id]
tokens=current_beam.tokens + [token_id]
if include_stop_str_in_output
else current_beam.tokens,
logprobs=current_beam.logprobs + [logprobs],
cum_logprob=candidate_logprob,
finish_reason="stop",
stop_reason=eos_token_id,
stop_reason=token_id,
)
)
else:
@@ -219,7 +223,7 @@ class BeamSearchOnlineMixin(ABC):
best_beams = sorted_completed[:beam_width]
for beam in best_beams:
if beam.tokens[-1] == eos_token_id and not ignore_eos:
if beam.tokens[-1] in eos_token_ids and not ignore_eos:
# Skip the eos token in the text.
tokens = beam.tokens[tokenized_length:-1]
else:
+20 -4
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
from vllm.inputs import (
DecoderOnlyEngineInput,
@@ -13,6 +14,7 @@ from vllm.inputs import (
)
from vllm.logprobs import Logprob
from vllm.lora.request import LoRARequest
from vllm.sampling_params import SamplingParams
@dataclass
@@ -137,7 +139,7 @@ class BeamSearchInstance:
def get_beam_search_score(
tokens: list[int],
cumulative_logprob: float,
eos_token_id: int,
eos_token_ids: set[int],
length_penalty: float = 1.0,
) -> float:
"""Calculate the beam search score with length penalty.
@@ -147,17 +149,31 @@ def get_beam_search_score(
https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
"""
seq_len = len(tokens)
if tokens[-1] == eos_token_id:
if tokens[-1] in eos_token_ids:
seq_len -= 1
# An aborted beam may contain only an EOS prompt token.
return cumulative_logprob / (max(seq_len, 1) ** length_penalty)
def create_sort_beams_key_function(eos_token_id: int, length_penalty: float):
def create_sort_beams_key_function(eos_token_ids: set[int], length_penalty: float):
def sort_beams_key(x: BeamSearchSequence) -> float:
return get_beam_search_score(
x.tokens, x.cum_logprob, eos_token_id, length_penalty
x.tokens, x.cum_logprob, eos_token_ids, length_penalty
)
return sort_beams_key
def get_beam_search_eos_token_ids(
generation_config: dict[str, Any], eos_token_id: int | None
) -> set[int]:
"""Return the token ids that end generation: the tokenizer EOS plus the
generation config's ``eos_token_id``.
This uses a separate SamplingParams so the params sent to the engine keep
their own ``stop_token_ids`` and are validated as before.
"""
params = SamplingParams()
params.update_from_generation_config(generation_config, eos_token_id)
return params.all_stop_token_ids