[Bugfix][Frontend] Handle aborted requests in beam search (#56249)

Signed-off-by: 子华 <huaxi.shx@alibaba-inc.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
shaohuaxi
2026-09-15 08:17:30 +00:00
committed by GitHub
co-authored by Codex
parent 2e2fdaa99a
commit ca67438c08
5 changed files with 162 additions and 9 deletions
+85
View File
@@ -6,13 +6,17 @@ Run `pytest tests/samplers/test_beam_search.py`.
"""
import json
from types import SimpleNamespace
from unittest.mock import Mock
import jsonschema
import pytest
from transformers import AutoModelForSeq2SeqLM
from vllm import CompletionOutput, RequestOutput
from vllm.assets.audio import AudioAsset
from vllm.entrypoints.llm import LLM
from vllm.logprobs import Logprob, SampleLogprobs
from vllm.platforms import current_platform
from vllm.sampling_params import BeamSearchParams, StructuredOutputsParams
@@ -45,6 +49,87 @@ MM_BEAM_WIDTHS = [2]
MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]
@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)])
@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]])
def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts(
monkeypatch,
abort_after: int,
prompt_token: int,
terminal_logprobs: SampleLogprobs | None,
) -> None:
"""Return tokens and scores from the last completed step for aborted prompts.
Other prompts in the batch continue generating.
"""
def run_requests(prompts, **kwargs):
results = []
for prompt in prompts:
tokens = prompt["prompt_token_ids"]
if tokens[0] == prompt_token:
assert len(tokens) <= abort_after + 1, "Continued after abort"
aborted = (
tokens[0] == prompt_token
and len(tokens) == abort_after + 1
and tokens[-1] != 12
)
results.append(
RequestOutput(
request_id="inner",
prompt=None,
prompt_token_ids=tokens,
prompt_logprobs=None,
finished=True,
outputs=[
CompletionOutput(
index=0,
text="",
token_ids=[] if aborted else [11],
cumulative_logprob=None,
logprobs=terminal_logprobs
if aborted
else [{11: Logprob(-0.1), 12: Logprob(-0.2)}],
finish_reason="abort" if aborted else "length",
)
],
)
)
return results
llm = LLM.__new__(LLM)
llm.llm_engine = Mock()
tokenizer = SimpleNamespace(
eos_token_id=0,
decode=lambda tokens, skip_special_tokens=False: " ".join(
str(token) for token in tokens if not skip_special_tokens or token != 0
),
)
llm.renderer = Mock(get_tokenizer=Mock(return_value=tokenizer))
monkeypatch.setattr(llm, "_preprocess_cmpl", lambda prompts: prompts)
monkeypatch.setattr(llm, "_render_and_run_requests", run_requests)
prompts = [
{"type": "token", "prompt_token_ids": [token]} for token in [prompt_token, 2]
]
params = BeamSearchParams(beam_width=2, max_tokens=3)
outputs = llm.beam_search(prompts, params)
aborted, normal = outputs
expected_tokens = (
[[prompt_token, 11], [prompt_token, 12]] if abort_after else [[prompt_token]]
)
assert [beam.tokens for beam in aborted.sequences] == expected_tokens
for beam in aborted.sequences:
assert beam.finish_reason == "abort"
assert beam.text == tokenizer.decode(
beam.tokens, skip_special_tokens=params.skip_special_tokens
)
assert len(beam.logprobs) == abort_after
assert aborted.sequences[0].cum_logprob == pytest.approx(-0.1 * abort_after)
assert len(normal.sequences) == 2
assert all(len(beam.tokens) == 4 for beam in normal.sequences)
assert all(beam.finish_reason != "abort" for beam in normal.sequences)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", MAX_TOKENS)
+53 -1
View File
@@ -5,7 +5,7 @@ import pytest
from vllm import CompletionOutput, RequestOutput
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
from vllm.logprobs import Logprob
from vllm.logprobs import Logprob, SampleLogprobs
from vllm.sampling_params import BeamSearchParams
@@ -108,3 +108,55 @@ async def test_beam_search_respects_skip_special_tokens(
assert outputs[0].outputs[0].text == expected_text
assert outputs[0].outputs[0].token_ids == [_Tokenizer.special_token_id]
@pytest.mark.asyncio
@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)])
@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]])
async def test_beam_search_abort_returns_partial_outputs(
monkeypatch,
abort_after: int,
prompt_token: int,
terminal_logprobs: SampleLogprobs | None,
) -> None:
"""Abort returns each beam's tokens and scores from the last completed step."""
calls = 0
async def generate(prompt, *args, **kwargs):
nonlocal calls
calls += 1
depth = len(prompt["prompt_token_ids"]) - 1
assert depth <= abort_after, "Beam search continued after abort"
result = await anext(_EngineClient().generate(prompt, *args, **kwargs))
output = result.outputs[0]
output.token_ids = [11]
output.logprobs = [{11: Logprob(-0.1), 12: Logprob(-0.2)}]
output.finish_reason = "length"
if depth == abort_after and prompt["prompt_token_ids"][-1] != 12:
output.token_ids = []
output.logprobs = terminal_logprobs
output.finish_reason = "abort"
yield result
serving = _Serving()
monkeypatch.setattr(serving.engine_client, "generate", generate)
prompt = {"type": "token", "prompt": "prompt", "prompt_token_ids": [prompt_token]}
outputs = [
output
async for output in serving.beam_search(
prompt, "request", BeamSearchParams(beam_width=2, max_tokens=4)
)
]
assert calls == 1 + 2 * abort_after
assert len(outputs) == 1
assert outputs[0].finished
assert outputs[0].request_id == "request"
expected_tokens = [[11], [12]] if abort_after else [[]]
assert [output.token_ids for output in outputs[0].outputs] == expected_tokens
for output in outputs[0].outputs:
assert output.finish_reason == "abort"
assert output.text == _Tokenizer().decode(list(output.token_ids))
assert output.logprobs is not None
assert len(output.logprobs) == abort_after
assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1 * abort_after)
@@ -293,6 +293,18 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
allowed_sets[i] = set(entry[1])
for (start, end), instance in zip(instance_start_and_end, instances_batch):
instance_output = output[start:end]
if any(
result is not None and result.outputs[0].finish_reason == "abort"
for result in instance_output
):
for beam, result in zip(all_beams[start:end], instance_output):
if result is not None:
beam.finish_reason = "abort"
instance.completed.append(beam)
instance.beams = []
continue
instance_new_beams = []
for i in range(start, end):
current_beam = all_beams[i]
@@ -302,8 +314,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
continue
if result.outputs[0].logprobs is not None:
# if logprobs is None, the sequence completed
# due to max-model-len or abortion.
logprobs = result.outputs[0].logprobs[0]
allowed = allowed_sets[i]
for token_id, logprob_obj in logprobs.items():
@@ -99,11 +99,7 @@ class BeamSearchOnlineMixin(ABC):
output = [x[0] for x in await asyncio.gather(*tasks)]
candidates = []
# Iterate through all beam inference results
for i, result in enumerate(output):
current_beam = all_beams[i]
for result in output:
# check for error finish reason and abort beam search
if result.outputs[0].finish_reason == "error":
# yield error output and terminate beam search
@@ -126,6 +122,15 @@ class BeamSearchOnlineMixin(ABC):
)
return
if any(result.outputs[0].finish_reason == "abort" for result in output):
for beam in all_beams:
beam.finish_reason = "abort"
break
candidates = []
for i, result in enumerate(output):
current_beam = all_beams[i]
if result.outputs[0].logprobs is not None:
logprobs = result.outputs[0].logprobs[0]
for token_id, logprob_obj in logprobs.items():
@@ -150,7 +150,8 @@ def get_beam_search_score(
if tokens[-1] == eos_token_id:
seq_len -= 1
return cumulative_logprob / (seq_len**length_penalty)
# An aborted beam may contain only an EOS prompt token.
return cumulative_logprob / (max(seq_len, 1) ** length_penalty)
def create_sort_beams_key_function(eos_token_id: int, length_penalty: float):