[Bugfix][Frontend] Handle aborted requests in beam search (#56249)
Signed-off-by: 子华 <huaxi.shx@alibaba-inc.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -6,13 +6,17 @@ Run `pytest tests/samplers/test_beam_search.py`.
|
||||
"""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
from transformers import AutoModelForSeq2SeqLM
|
||||
|
||||
from vllm import CompletionOutput, RequestOutput
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.entrypoints.llm import LLM
|
||||
from vllm.logprobs import Logprob, SampleLogprobs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import BeamSearchParams, StructuredOutputsParams
|
||||
|
||||
@@ -45,6 +49,87 @@ MM_BEAM_WIDTHS = [2]
|
||||
MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)])
|
||||
@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]])
|
||||
def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts(
|
||||
monkeypatch,
|
||||
abort_after: int,
|
||||
prompt_token: int,
|
||||
terminal_logprobs: SampleLogprobs | None,
|
||||
) -> None:
|
||||
"""Return tokens and scores from the last completed step for aborted prompts.
|
||||
|
||||
Other prompts in the batch continue generating.
|
||||
"""
|
||||
|
||||
def run_requests(prompts, **kwargs):
|
||||
results = []
|
||||
for prompt in prompts:
|
||||
tokens = prompt["prompt_token_ids"]
|
||||
if tokens[0] == prompt_token:
|
||||
assert len(tokens) <= abort_after + 1, "Continued after abort"
|
||||
aborted = (
|
||||
tokens[0] == prompt_token
|
||||
and len(tokens) == abort_after + 1
|
||||
and tokens[-1] != 12
|
||||
)
|
||||
results.append(
|
||||
RequestOutput(
|
||||
request_id="inner",
|
||||
prompt=None,
|
||||
prompt_token_ids=tokens,
|
||||
prompt_logprobs=None,
|
||||
finished=True,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[] if aborted else [11],
|
||||
cumulative_logprob=None,
|
||||
logprobs=terminal_logprobs
|
||||
if aborted
|
||||
else [{11: Logprob(-0.1), 12: Logprob(-0.2)}],
|
||||
finish_reason="abort" if aborted else "length",
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
llm = LLM.__new__(LLM)
|
||||
llm.llm_engine = Mock()
|
||||
tokenizer = SimpleNamespace(
|
||||
eos_token_id=0,
|
||||
decode=lambda tokens, skip_special_tokens=False: " ".join(
|
||||
str(token) for token in tokens if not skip_special_tokens or token != 0
|
||||
),
|
||||
)
|
||||
llm.renderer = Mock(get_tokenizer=Mock(return_value=tokenizer))
|
||||
monkeypatch.setattr(llm, "_preprocess_cmpl", lambda prompts: prompts)
|
||||
monkeypatch.setattr(llm, "_render_and_run_requests", run_requests)
|
||||
prompts = [
|
||||
{"type": "token", "prompt_token_ids": [token]} for token in [prompt_token, 2]
|
||||
]
|
||||
params = BeamSearchParams(beam_width=2, max_tokens=3)
|
||||
outputs = llm.beam_search(prompts, params)
|
||||
|
||||
aborted, normal = outputs
|
||||
expected_tokens = (
|
||||
[[prompt_token, 11], [prompt_token, 12]] if abort_after else [[prompt_token]]
|
||||
)
|
||||
assert [beam.tokens for beam in aborted.sequences] == expected_tokens
|
||||
for beam in aborted.sequences:
|
||||
assert beam.finish_reason == "abort"
|
||||
assert beam.text == tokenizer.decode(
|
||||
beam.tokens, skip_special_tokens=params.skip_special_tokens
|
||||
)
|
||||
assert len(beam.logprobs) == abort_after
|
||||
assert aborted.sequences[0].cum_logprob == pytest.approx(-0.1 * abort_after)
|
||||
assert len(normal.sequences) == 2
|
||||
assert all(len(beam.tokens) == 4 for beam in normal.sequences)
|
||||
assert all(beam.finish_reason != "abort" for beam in normal.sequences)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", MAX_TOKENS)
|
||||
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
|
||||
from vllm import CompletionOutput, RequestOutput
|
||||
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.logprobs import Logprob, SampleLogprobs
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
|
||||
|
||||
@@ -108,3 +108,55 @@ async def test_beam_search_respects_skip_special_tokens(
|
||||
|
||||
assert outputs[0].outputs[0].text == expected_text
|
||||
assert outputs[0].outputs[0].token_ids == [_Tokenizer.special_token_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)])
|
||||
@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]])
|
||||
async def test_beam_search_abort_returns_partial_outputs(
|
||||
monkeypatch,
|
||||
abort_after: int,
|
||||
prompt_token: int,
|
||||
terminal_logprobs: SampleLogprobs | None,
|
||||
) -> None:
|
||||
"""Abort returns each beam's tokens and scores from the last completed step."""
|
||||
calls = 0
|
||||
|
||||
async def generate(prompt, *args, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
depth = len(prompt["prompt_token_ids"]) - 1
|
||||
assert depth <= abort_after, "Beam search continued after abort"
|
||||
result = await anext(_EngineClient().generate(prompt, *args, **kwargs))
|
||||
output = result.outputs[0]
|
||||
output.token_ids = [11]
|
||||
output.logprobs = [{11: Logprob(-0.1), 12: Logprob(-0.2)}]
|
||||
output.finish_reason = "length"
|
||||
if depth == abort_after and prompt["prompt_token_ids"][-1] != 12:
|
||||
output.token_ids = []
|
||||
output.logprobs = terminal_logprobs
|
||||
output.finish_reason = "abort"
|
||||
yield result
|
||||
|
||||
serving = _Serving()
|
||||
monkeypatch.setattr(serving.engine_client, "generate", generate)
|
||||
prompt = {"type": "token", "prompt": "prompt", "prompt_token_ids": [prompt_token]}
|
||||
outputs = [
|
||||
output
|
||||
async for output in serving.beam_search(
|
||||
prompt, "request", BeamSearchParams(beam_width=2, max_tokens=4)
|
||||
)
|
||||
]
|
||||
|
||||
assert calls == 1 + 2 * abort_after
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].finished
|
||||
assert outputs[0].request_id == "request"
|
||||
expected_tokens = [[11], [12]] if abort_after else [[]]
|
||||
assert [output.token_ids for output in outputs[0].outputs] == expected_tokens
|
||||
for output in outputs[0].outputs:
|
||||
assert output.finish_reason == "abort"
|
||||
assert output.text == _Tokenizer().decode(list(output.token_ids))
|
||||
assert output.logprobs is not None
|
||||
assert len(output.logprobs) == abort_after
|
||||
assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1 * abort_after)
|
||||
|
||||
@@ -293,6 +293,18 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
allowed_sets[i] = set(entry[1])
|
||||
|
||||
for (start, end), instance in zip(instance_start_and_end, instances_batch):
|
||||
instance_output = output[start:end]
|
||||
if any(
|
||||
result is not None and result.outputs[0].finish_reason == "abort"
|
||||
for result in instance_output
|
||||
):
|
||||
for beam, result in zip(all_beams[start:end], instance_output):
|
||||
if result is not None:
|
||||
beam.finish_reason = "abort"
|
||||
instance.completed.append(beam)
|
||||
instance.beams = []
|
||||
continue
|
||||
|
||||
instance_new_beams = []
|
||||
for i in range(start, end):
|
||||
current_beam = all_beams[i]
|
||||
@@ -302,8 +314,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
continue
|
||||
|
||||
if result.outputs[0].logprobs is not None:
|
||||
# if logprobs is None, the sequence completed
|
||||
# due to max-model-len or abortion.
|
||||
logprobs = result.outputs[0].logprobs[0]
|
||||
allowed = allowed_sets[i]
|
||||
for token_id, logprob_obj in logprobs.items():
|
||||
|
||||
@@ -99,11 +99,7 @@ class BeamSearchOnlineMixin(ABC):
|
||||
|
||||
output = [x[0] for x in await asyncio.gather(*tasks)]
|
||||
|
||||
candidates = []
|
||||
# Iterate through all beam inference results
|
||||
for i, result in enumerate(output):
|
||||
current_beam = all_beams[i]
|
||||
|
||||
for result in output:
|
||||
# check for error finish reason and abort beam search
|
||||
if result.outputs[0].finish_reason == "error":
|
||||
# yield error output and terminate beam search
|
||||
@@ -126,6 +122,15 @@ class BeamSearchOnlineMixin(ABC):
|
||||
)
|
||||
return
|
||||
|
||||
if any(result.outputs[0].finish_reason == "abort" for result in output):
|
||||
for beam in all_beams:
|
||||
beam.finish_reason = "abort"
|
||||
break
|
||||
|
||||
candidates = []
|
||||
for i, result in enumerate(output):
|
||||
current_beam = all_beams[i]
|
||||
|
||||
if result.outputs[0].logprobs is not None:
|
||||
logprobs = result.outputs[0].logprobs[0]
|
||||
for token_id, logprob_obj in logprobs.items():
|
||||
|
||||
@@ -150,7 +150,8 @@ def get_beam_search_score(
|
||||
if tokens[-1] == eos_token_id:
|
||||
seq_len -= 1
|
||||
|
||||
return cumulative_logprob / (seq_len**length_penalty)
|
||||
# An aborted beam may contain only an EOS prompt token.
|
||||
return cumulative_logprob / (max(seq_len, 1) ** length_penalty)
|
||||
|
||||
|
||||
def create_sort_beams_key_function(eos_token_id: int, length_penalty: float):
|
||||
|
||||
Reference in New Issue
Block a user