From ca67438c088d5ada7da901b77f9aeaebc1b9542c Mon Sep 17 00:00:00 2001 From: shaohuaxi Date: Tue, 15 Sep 2026 01:17:30 -0700 Subject: [PATCH] [Bugfix][Frontend] Handle aborted requests in beam search (#56249) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 子华 Co-authored-by: Codex --- tests/samplers/test_beam_search.py | 85 +++++++++++++++++++ tests/samplers/test_beam_search_online.py | 54 +++++++++++- .../generate/beam_search/offline.py | 14 ++- .../generate/beam_search/online.py | 15 ++-- .../entrypoints/generate/beam_search/utils.py | 3 +- 5 files changed, 162 insertions(+), 9 deletions(-) diff --git a/tests/samplers/test_beam_search.py b/tests/samplers/test_beam_search.py index 91dee5634b..8bcf4c48ab 100644 --- a/tests/samplers/test_beam_search.py +++ b/tests/samplers/test_beam_search.py @@ -6,13 +6,17 @@ Run `pytest tests/samplers/test_beam_search.py`. """ import json +from types import SimpleNamespace +from unittest.mock import Mock import jsonschema import pytest from transformers import AutoModelForSeq2SeqLM +from vllm import CompletionOutput, RequestOutput from vllm.assets.audio import AudioAsset from vllm.entrypoints.llm import LLM +from vllm.logprobs import Logprob, SampleLogprobs from vllm.platforms import current_platform from vllm.sampling_params import BeamSearchParams, StructuredOutputsParams @@ -45,6 +49,87 @@ MM_BEAM_WIDTHS = [2] MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"] +@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)]) +@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]]) +def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts( + monkeypatch, + abort_after: int, + prompt_token: int, + terminal_logprobs: SampleLogprobs | None, +) -> None: + """Return tokens and scores from the last completed step for aborted prompts. + + Other prompts in the batch continue generating. + """ + + def run_requests(prompts, **kwargs): + results = [] + for prompt in prompts: + tokens = prompt["prompt_token_ids"] + if tokens[0] == prompt_token: + assert len(tokens) <= abort_after + 1, "Continued after abort" + aborted = ( + tokens[0] == prompt_token + and len(tokens) == abort_after + 1 + and tokens[-1] != 12 + ) + results.append( + RequestOutput( + request_id="inner", + prompt=None, + prompt_token_ids=tokens, + prompt_logprobs=None, + finished=True, + outputs=[ + CompletionOutput( + index=0, + text="", + token_ids=[] if aborted else [11], + cumulative_logprob=None, + logprobs=terminal_logprobs + if aborted + else [{11: Logprob(-0.1), 12: Logprob(-0.2)}], + finish_reason="abort" if aborted else "length", + ) + ], + ) + ) + return results + + llm = LLM.__new__(LLM) + llm.llm_engine = Mock() + tokenizer = SimpleNamespace( + eos_token_id=0, + decode=lambda tokens, skip_special_tokens=False: " ".join( + str(token) for token in tokens if not skip_special_tokens or token != 0 + ), + ) + llm.renderer = Mock(get_tokenizer=Mock(return_value=tokenizer)) + monkeypatch.setattr(llm, "_preprocess_cmpl", lambda prompts: prompts) + monkeypatch.setattr(llm, "_render_and_run_requests", run_requests) + prompts = [ + {"type": "token", "prompt_token_ids": [token]} for token in [prompt_token, 2] + ] + params = BeamSearchParams(beam_width=2, max_tokens=3) + outputs = llm.beam_search(prompts, params) + + aborted, normal = outputs + expected_tokens = ( + [[prompt_token, 11], [prompt_token, 12]] if abort_after else [[prompt_token]] + ) + assert [beam.tokens for beam in aborted.sequences] == expected_tokens + for beam in aborted.sequences: + assert beam.finish_reason == "abort" + assert beam.text == tokenizer.decode( + beam.tokens, skip_special_tokens=params.skip_special_tokens + ) + assert len(beam.logprobs) == abort_after + assert aborted.sequences[0].cum_logprob == pytest.approx(-0.1 * abort_after) + assert len(normal.sequences) == 2 + assert all(len(beam.tokens) == 4 for beam in normal.sequences) + assert all(beam.finish_reason != "abort" for beam in normal.sequences) + + @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("dtype", ["half"]) @pytest.mark.parametrize("max_tokens", MAX_TOKENS) diff --git a/tests/samplers/test_beam_search_online.py b/tests/samplers/test_beam_search_online.py index 102e66dcbe..ccb40cc659 100644 --- a/tests/samplers/test_beam_search_online.py +++ b/tests/samplers/test_beam_search_online.py @@ -5,7 +5,7 @@ import pytest from vllm import CompletionOutput, RequestOutput from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin -from vllm.logprobs import Logprob +from vllm.logprobs import Logprob, SampleLogprobs from vllm.sampling_params import BeamSearchParams @@ -108,3 +108,55 @@ async def test_beam_search_respects_skip_special_tokens( assert outputs[0].outputs[0].text == expected_text assert outputs[0].outputs[0].token_ids == [_Tokenizer.special_token_id] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)]) +@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]]) +async def test_beam_search_abort_returns_partial_outputs( + monkeypatch, + abort_after: int, + prompt_token: int, + terminal_logprobs: SampleLogprobs | None, +) -> None: + """Abort returns each beam's tokens and scores from the last completed step.""" + calls = 0 + + async def generate(prompt, *args, **kwargs): + nonlocal calls + calls += 1 + depth = len(prompt["prompt_token_ids"]) - 1 + assert depth <= abort_after, "Beam search continued after abort" + result = await anext(_EngineClient().generate(prompt, *args, **kwargs)) + output = result.outputs[0] + output.token_ids = [11] + output.logprobs = [{11: Logprob(-0.1), 12: Logprob(-0.2)}] + output.finish_reason = "length" + if depth == abort_after and prompt["prompt_token_ids"][-1] != 12: + output.token_ids = [] + output.logprobs = terminal_logprobs + output.finish_reason = "abort" + yield result + + serving = _Serving() + monkeypatch.setattr(serving.engine_client, "generate", generate) + prompt = {"type": "token", "prompt": "prompt", "prompt_token_ids": [prompt_token]} + outputs = [ + output + async for output in serving.beam_search( + prompt, "request", BeamSearchParams(beam_width=2, max_tokens=4) + ) + ] + + assert calls == 1 + 2 * abort_after + assert len(outputs) == 1 + assert outputs[0].finished + assert outputs[0].request_id == "request" + expected_tokens = [[11], [12]] if abort_after else [[]] + assert [output.token_ids for output in outputs[0].outputs] == expected_tokens + for output in outputs[0].outputs: + assert output.finish_reason == "abort" + assert output.text == _Tokenizer().decode(list(output.token_ids)) + assert output.logprobs is not None + assert len(output.logprobs) == abort_after + assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1 * abort_after) diff --git a/vllm/entrypoints/generate/beam_search/offline.py b/vllm/entrypoints/generate/beam_search/offline.py index b919435f65..73b9ebe06b 100644 --- a/vllm/entrypoints/generate/beam_search/offline.py +++ b/vllm/entrypoints/generate/beam_search/offline.py @@ -293,6 +293,18 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin): allowed_sets[i] = set(entry[1]) for (start, end), instance in zip(instance_start_and_end, instances_batch): + instance_output = output[start:end] + if any( + result is not None and result.outputs[0].finish_reason == "abort" + for result in instance_output + ): + for beam, result in zip(all_beams[start:end], instance_output): + if result is not None: + beam.finish_reason = "abort" + instance.completed.append(beam) + instance.beams = [] + continue + instance_new_beams = [] for i in range(start, end): current_beam = all_beams[i] @@ -302,8 +314,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin): continue if result.outputs[0].logprobs is not None: - # if logprobs is None, the sequence completed - # due to max-model-len or abortion. logprobs = result.outputs[0].logprobs[0] allowed = allowed_sets[i] for token_id, logprob_obj in logprobs.items(): diff --git a/vllm/entrypoints/generate/beam_search/online.py b/vllm/entrypoints/generate/beam_search/online.py index ac1a585e8f..da02b974c0 100644 --- a/vllm/entrypoints/generate/beam_search/online.py +++ b/vllm/entrypoints/generate/beam_search/online.py @@ -99,11 +99,7 @@ class BeamSearchOnlineMixin(ABC): output = [x[0] for x in await asyncio.gather(*tasks)] - candidates = [] - # Iterate through all beam inference results - for i, result in enumerate(output): - current_beam = all_beams[i] - + for result in output: # check for error finish reason and abort beam search if result.outputs[0].finish_reason == "error": # yield error output and terminate beam search @@ -126,6 +122,15 @@ class BeamSearchOnlineMixin(ABC): ) return + if any(result.outputs[0].finish_reason == "abort" for result in output): + for beam in all_beams: + beam.finish_reason = "abort" + break + + candidates = [] + for i, result in enumerate(output): + current_beam = all_beams[i] + if result.outputs[0].logprobs is not None: logprobs = result.outputs[0].logprobs[0] for token_id, logprob_obj in logprobs.items(): diff --git a/vllm/entrypoints/generate/beam_search/utils.py b/vllm/entrypoints/generate/beam_search/utils.py index 47f69cbf7a..61f97963b6 100644 --- a/vllm/entrypoints/generate/beam_search/utils.py +++ b/vllm/entrypoints/generate/beam_search/utils.py @@ -150,7 +150,8 @@ def get_beam_search_score( if tokens[-1] == eos_token_id: seq_len -= 1 - return cumulative_logprob / (seq_len**length_penalty) + # An aborted beam may contain only an EOS prompt token. + return cumulative_logprob / (max(seq_len, 1) ** length_penalty) def create_sort_beams_key_function(eos_token_id: int, length_penalty: float):