[Bugfix][Frontend] Stop beam search on generation config EOS tokens (#62072)
pre-commit / pre-run-check (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s

Signed-off-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
Co-authored-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
This commit is contained in:
alp
2026-10-18 16:49:42 +00:00
committed by GitHub
co-authored by mottopanikeiku
parent fe8f2140ee
commit ff01a6b14f
5 changed files with 110 additions and 16 deletions
+1
View File
@@ -99,6 +99,7 @@ def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts(
llm = LLM.__new__(LLM)
llm.llm_engine = Mock()
llm.llm_engine.input_processor.generation_config_fields = {}
tokenizer = SimpleNamespace(
eos_token_id=0,
decode=lambda tokens, skip_special_tokens=False: " ".join(
+70 -1
View File
@@ -67,6 +67,7 @@ class _VllmConfig:
class _InputProcessor:
vllm_config = _VllmConfig()
generation_config_fields: dict[str, Any] = {}
def resolve_watermarking(self, params):
return self.vllm_config._check_supports_watermarking(params)
@@ -128,7 +129,7 @@ class _OfflineServing(BeamSearchOfflineMixin):
self,
instances_batch: list[BeamSearchInstance],
base_sampling_params: SamplingParams,
eos_token_id: int | None,
eos_token_ids: set[int],
ignore_eos: bool,
beam_width: int,
sort_beams_key: Callable[..., Any],
@@ -140,6 +141,74 @@ class _OfflineServing(BeamSearchOfflineMixin):
return True
@pytest.mark.asyncio
@pytest.mark.parametrize("offline", [False, True])
@pytest.mark.parametrize("config_eos", [None, 11, [0, 11]])
@pytest.mark.parametrize("ignore_eos", [False, True])
async def test_beam_search_respects_generation_config_eos(
monkeypatch, offline: bool, config_eos: int | list[int] | None, ignore_eos: bool
) -> None:
"""Model-specific end-of-turn tokens must finish beams like tokenizer EOS."""
monkeypatch.setattr(
_InputProcessor, "generation_config_fields", {"eos_token_id": config_eos}
)
prompt: TokensInput = {
"type": "token",
"prompt_token_ids": [1],
}
params = BeamSearchParams(
beam_width=1,
max_tokens=3,
ignore_eos=ignore_eos,
include_stop_str_in_output=True,
watermarking=False,
)
result = await anext(
_EngineClient().generate(prompt, SamplingParams(watermarking=False))
)
result.outputs[0].logprobs = [{11: Logprob(-0.1)}]
should_stop = config_eos is not None and not ignore_eos
expected_tokens = [11] if should_stop else [11, 11, 11]
engine_params: list[SamplingParams] = []
if offline:
offline_serving = _OfflineServing()
monkeypatch.setattr(
_OfflineServing,
"_beam_search_step",
BeamSearchOfflineMixin._beam_search_step,
)
def render_and_run_requests(*, params, **kwargs):
engine_params.extend(params)
return [result]
monkeypatch.setattr(
offline_serving, "_render_and_run_requests", render_and_run_requests
)
beam = offline_serving.beam_search(cast(list, [prompt]), params)[0].sequences[0]
assert beam.tokens == [1, *expected_tokens]
else:
async def generate(engine_prompt, sampling_params, *args, **kwargs):
engine_params.append(sampling_params)
yield result
online_serving = _AsyncServing()
monkeypatch.setattr(online_serving.engine_client, "generate", generate)
output = (
await anext(online_serving.beam_search(prompt, "request", params))
).outputs[0]
assert output.token_ids == expected_tokens
assert output.finish_reason == ("stop" if should_stop else "length")
assert output.stop_reason == (11 if should_stop else None)
# The engine validates stop_token_ids against the vocab before it applies
# the generation config, so the config EOS ids must not be copied into them.
assert engine_params
assert all(not p.stop_token_ids for p in engine_params)
@pytest.mark.asyncio
async def test_beam_search_handles_extra_logprob_candidates() -> None:
prompt: TokensInput = {