[Core] Support structured outputs for beam search (#35022)

Signed-off-by: Guan-Ming (Wesley) Chiu <guanmingchiu@gmail.com>
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
This commit is contained in:
Guan-Ming (Wesley) Chiu
2026-06-12 06:56:25 -07:00
committed by GitHub
parent fbc3a1907a
commit c7aa3d2630
3 changed files with 405 additions and 70 deletions
+63
View File
@@ -5,11 +5,16 @@
Run `pytest tests/samplers/test_beam_search.py`.
"""
import json
import jsonschema
import pytest
from transformers import AutoModelForSeq2SeqLM
from vllm.assets.audio import AudioAsset
from vllm.entrypoints.llm import LLM
from vllm.platforms import current_platform
from vllm.sampling_params import BeamSearchParams, StructuredOutputsParams
# Extra engine kwargs needed for numerically deterministic beam search.
# On ROCm, floating-point reductions in attention and GEMM kernels are
@@ -223,3 +228,61 @@ def test_beam_search_passes_multimodal_data(
# NOTE: encoder/decoder tests are currently located under
# tests/models/multimodal/generation/test_whisper.py
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("beam_width", BEAM_WIDTHS)
def test_beam_search_structured_output(
model: str,
dtype: str,
beam_width: int,
) -> None:
"""Ensure beam search with structured output produces valid JSON."""
json_schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"},
},
"required": ["name", "age"],
"additionalProperties": False,
}
llm = LLM(
model=model,
dtype=dtype,
max_model_len=512,
structured_outputs_config=dict(
backend="xgrammar",
disable_any_whitespace=True,
),
**(dict(enforce_eager=True) | EXTRA_ENGINE_KWARGS),
)
params = BeamSearchParams(
beam_width=beam_width,
max_tokens=64,
structured_outputs=StructuredOutputsParams(json=json_schema),
)
prompts = [
"Generate a JSON object for a person with name and age:",
]
outputs = llm.beam_search(prompts, params)
assert len(outputs) == len(prompts)
for output in outputs:
assert len(output.sequences) > 0
for seq in output.sequences:
assert seq.text is not None
print(f"Full text: {seq.text!r}")
# seq.text includes the prompt, extract generated JSON.
gen_start = seq.text.find("{")
assert gen_start != -1, f"No JSON found in output: {seq.text!r}"
generated = seq.text[gen_start:]
generated = generated.replace("</s>", "").strip()
print(f"Generated JSON: {generated!r}")
parsed = json.loads(generated)
jsonschema.validate(instance=parsed, schema=json_schema)
+341 -70
View File
@@ -2,14 +2,24 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from collections.abc import Callable, Sequence
import torch
from tqdm import tqdm
from vllm import RequestOutput, TextPrompt, TokensPrompt
from vllm.entrypoints.offline_utils import OfflineInferenceMixin
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import (
BeamSearchParams,
SamplingParams,
StructuredOutputsParams,
)
from vllm.tokenizers import TokenizerLike
from vllm.v1.structured_output.backend_types import StructuredOutputBackend
from vllm.v1.structured_output.request import get_structured_output_key
from .utils import (
BeamSearchInstance,
@@ -20,6 +30,27 @@ from .utils import (
logger = init_logger(__name__)
# Engine-side cap on `SamplingParams.allowed_token_ids`; keep in sync with
# MAX_NUM_ALLOWED_TOKEN_IDS in vllm/v1/worker/gpu/sample/logit_bias.py.
_MAX_NUM_ALLOWED_TOKEN_IDS = 1024
_bitmask_cache: dict[int, tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
def _bitmask_to_token_ids(bitmask_row: torch.Tensor, vocab_size: int) -> list[int]:
"""Convert a packed int32 bitmask row to a list of allowed token IDs."""
if vocab_size not in _bitmask_cache:
indices = torch.arange(vocab_size)
_bitmask_cache[vocab_size] = (
indices,
indices >> 5, # i // 32
indices & 31, # i % 32
)
indices, word_indices, bit_indices = _bitmask_cache[vocab_size]
mask = ((bitmask_row[word_indices] >> bit_indices) & 1).bool()
return indices[mask].tolist()
class BeamSearchOfflineMixin(OfflineInferenceMixin):
"""Offline inference for beam search"""
@@ -69,10 +100,22 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
if concurrency_limit is None:
concurrency_limit = len(engine_inputs)
structured_output_backend: StructuredOutputBackend | None = None
structured_output_key = None
structured_output_bitmask = None
if params.structured_outputs is not None:
(
structured_output_backend,
structured_output_key,
structured_output_bitmask,
) = self._init_beam_search_structured_output(
params.structured_outputs, tokenizer
)
# generate 2 * beam_width candidates at each step
# following the huggingface transformers implementation
# at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
sampling_params = SamplingParams(
base_sampling_params = SamplingParams(
logprobs=2 * beam_width,
max_tokens=1,
temperature=temperature,
@@ -94,77 +137,43 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
),
)
for prompt_start in range(0, len(instances), concurrency_limit):
instances_batch = instances[prompt_start : prompt_start + concurrency_limit]
try:
for prompt_start in range(0, len(instances), concurrency_limit):
instances_batch = instances[
prompt_start : prompt_start + concurrency_limit
]
token_iter = range(max_tokens)
if use_tqdm:
token_iter = tqdm(
token_iter, desc="Beam search", unit="token", unit_scale=False
)
logger.warning(
"The progress bar shows the upper bound on token steps and "
"may finish early due to stopping conditions. It does not "
"reflect instance-level progress."
)
for _ in token_iter:
all_beams: list[BeamSearchSequence] = list(
sum((instance.beams for instance in instances_batch), [])
)
pos = [0] + list(
itertools.accumulate(
len(instance.beams) for instance in instances_batch
token_iter = range(max_tokens)
if use_tqdm:
token_iter = tqdm(
token_iter,
desc="Beam search",
unit="token",
unit_scale=False,
)
)
instance_start_and_end: list[tuple[int, int]] = list(
zip(pos[:-1], pos[1:])
)
if len(all_beams) == 0:
break
# only runs for one step
# we don't need to use tqdm here
output = self._render_and_run_requests(
prompts=(beam.get_prompt() for beam in all_beams),
params=self._params_to_seq(sampling_params, len(all_beams)),
output_type=RequestOutput,
lora_requests=[beam.lora_request for beam in all_beams],
use_tqdm=False,
)
for (start, end), instance in zip(
instance_start_and_end, instances_batch
):
instance_new_beams = []
for i in range(start, end):
current_beam = all_beams[i]
result = output[i]
if result.outputs[0].logprobs is not None:
# if `result.outputs[0].logprobs` is None, it means
# the sequence is completed because of the
# max-model-len or abortion. we don't need to add
# it to the new beams.
logprobs = result.outputs[0].logprobs[0]
for token_id, logprob_obj in logprobs.items():
new_beam = BeamSearchSequence(
current_beam.orig_prompt,
tokens=current_beam.tokens + [token_id],
logprobs=current_beam.logprobs + [logprobs],
lora_request=current_beam.lora_request,
cum_logprob=current_beam.cum_logprob
+ logprob_obj.logprob,
)
if token_id == eos_token_id and not ignore_eos:
instance.completed.append(new_beam)
else:
instance_new_beams.append(new_beam)
sorted_beams = sorted(
instance_new_beams, key=sort_beams_key, reverse=True
logger.warning(
"The progress bar shows the upper bound on token "
"steps and may finish early due to stopping "
"conditions. It does not reflect instance-level "
"progress."
)
instance.beams = sorted_beams[:beam_width]
for _ in token_iter:
should_stop = self._beam_search_step(
instances_batch=instances_batch,
base_sampling_params=base_sampling_params,
eos_token_id=eos_token_id,
ignore_eos=ignore_eos,
beam_width=beam_width,
sort_beams_key=sort_beams_key,
structured_output_backend=structured_output_backend,
structured_output_key=structured_output_key,
structured_output_bitmask=structured_output_bitmask,
)
if should_stop:
break
finally:
if structured_output_backend is not None:
structured_output_backend.destroy()
outputs = []
for instance in instances:
@@ -180,3 +189,265 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
outputs.append(BeamSearchOutput(sequences=best_beams))
return outputs
def _beam_search_step(
self,
instances_batch: list[BeamSearchInstance],
base_sampling_params: SamplingParams,
eos_token_id: int | None,
ignore_eos: bool,
beam_width: int,
sort_beams_key: Callable,
structured_output_backend: StructuredOutputBackend | None,
structured_output_key: tuple | None,
structured_output_bitmask: torch.Tensor | None,
) -> bool:
"""Run one token step of beam search across a batch of instances.
Returns True if all beams are exhausted and search should stop.
"""
all_beams: list[BeamSearchSequence] = list(
sum((instance.beams for instance in instances_batch), [])
)
pos = [0] + list(
itertools.accumulate(len(instance.beams) for instance in instances_batch)
)
instance_start_and_end: list[tuple[int, int]] = list(zip(pos[:-1], pos[1:]))
if len(all_beams) == 0:
return True
if structured_output_backend is not None:
assert (
structured_output_key is not None
and structured_output_bitmask is not None
)
beam_entries = self._build_beam_sampling_params(
all_beams,
base_sampling_params,
structured_output_backend,
structured_output_key,
structured_output_bitmask,
)
active_indices = [
i for i, entry in enumerate(beam_entries) if entry is not None
]
for i, entry in enumerate(beam_entries):
if entry is None:
beam = all_beams[i]
assert beam.orig_prompt["type"] != "enc_dec"
prompt_len = len(beam.orig_prompt["prompt_token_ids"])
if len(beam.tokens) > prompt_len:
for (s, e), inst in zip(
instance_start_and_end,
instances_batch,
):
if s <= i < e:
inst.completed.append(beam)
break
if not active_indices:
return True
active_beams = [all_beams[i] for i in active_indices]
active_params: Sequence[SamplingParams | PoolingParams] = [
beam_entries[i][0] # type: ignore[index]
for i in active_indices
]
else:
active_indices = list(range(len(all_beams)))
active_beams = all_beams
active_params = self._params_to_seq( # type: ignore[assignment]
base_sampling_params, len(all_beams)
)
# only runs for one step
# we don't need to use tqdm here
active_output = self._render_and_run_requests(
prompts=(beam.get_prompt() for beam in active_beams),
params=active_params,
output_type=RequestOutput,
lora_requests=[beam.lora_request for beam in active_beams],
use_tqdm=False,
)
output: list[RequestOutput | None] = [None] * len(all_beams)
for idx, active_idx in enumerate(active_indices):
output[active_idx] = active_output[idx]
# Logprobs are computed from raw logits before
# allowed_token_ids masking, so they may contain
# tokens outside the grammar's allowed set. This filtering is also
# the only grammar enforcement for beams whose allowed set exceeds
# the engine-side allowed_token_ids cap.
allowed_sets: list[set[int] | None] = [None] * len(all_beams)
if structured_output_backend is not None:
for i, entry in enumerate(beam_entries):
if entry is not None:
allowed_sets[i] = set(entry[1])
for (start, end), instance in zip(instance_start_and_end, instances_batch):
instance_new_beams = []
for i in range(start, end):
current_beam = all_beams[i]
result = output[i]
if result is None:
continue
if result.outputs[0].logprobs is not None:
# if logprobs is None, the sequence completed
# due to max-model-len or abortion.
logprobs = result.outputs[0].logprobs[0]
allowed = allowed_sets[i]
for token_id, logprob_obj in logprobs.items():
if allowed is not None and token_id not in allowed:
continue
new_beam = BeamSearchSequence(
current_beam.orig_prompt,
tokens=current_beam.tokens + [token_id],
logprobs=current_beam.logprobs + [logprobs],
lora_request=current_beam.lora_request,
cum_logprob=current_beam.cum_logprob + logprob_obj.logprob,
)
if token_id == eos_token_id and not ignore_eos:
instance.completed.append(new_beam)
else:
instance_new_beams.append(new_beam)
sorted_beams = sorted(
instance_new_beams,
key=sort_beams_key,
reverse=True,
)
instance.beams = sorted_beams[:beam_width]
return False
def _init_beam_search_structured_output(
self,
structured_outputs: StructuredOutputsParams,
tokenizer: TokenizerLike,
) -> tuple[StructuredOutputBackend, tuple, torch.Tensor]:
"""Initialize the structured output backend for beam search."""
vllm_config = self.llm_engine.vllm_config
so_config = vllm_config.structured_outputs_config
if so_config is None:
raise ValueError(
"structured_outputs_config is required for beam search "
"with structured outputs"
)
# Resolve the backend name from engine config if not already set.
if not structured_outputs._backend:
structured_outputs._backend = so_config.backend
backend_name = structured_outputs._backend
vocab_size = self.model_config.get_vocab_size()
backend: StructuredOutputBackend
if backend_name == "xgrammar":
from vllm.v1.structured_output.backend_xgrammar import (
XgrammarBackend,
)
backend = XgrammarBackend(
vllm_config=vllm_config,
tokenizer=tokenizer,
vocab_size=vocab_size,
)
elif backend_name == "guidance":
from vllm.v1.structured_output.backend_guidance import (
GuidanceBackend,
)
backend = GuidanceBackend(
vllm_config=vllm_config,
tokenizer=tokenizer,
vocab_size=vocab_size,
)
elif backend_name == "outlines":
from vllm.v1.structured_output.backend_outlines import (
OutlinesBackend,
)
backend = OutlinesBackend(
vllm_config=vllm_config,
tokenizer=tokenizer,
vocab_size=vocab_size,
)
elif backend_name == "lm-format-enforcer":
from vllm.v1.structured_output.backend_lm_format_enforcer import (
LMFormatEnforcerBackend,
)
backend = LMFormatEnforcerBackend(
vllm_config=vllm_config,
tokenizer=tokenizer,
vocab_size=vocab_size,
)
else:
raise ValueError(f"Unsupported structured output backend: {backend_name}")
structured_output_key = get_structured_output_key(structured_outputs)
bitmask = backend.allocate_token_bitmask(1)
return backend, structured_output_key, bitmask
def _build_beam_sampling_params(
self,
beams: list[BeamSearchSequence],
base_params: SamplingParams,
backend: StructuredOutputBackend,
structured_output_key: tuple,
bitmask: torch.Tensor,
) -> list[tuple[SamplingParams, list[int]] | None]:
"""Build per-beam SamplingParams and allowed token IDs from grammar.
Returns None for beams where the grammar has terminated.
"""
vocab_size = self.model_config.get_vocab_size()
request_type, grammar_spec = structured_output_key
result: list[tuple[SamplingParams, list[int]] | None] = []
for beam in beams:
# Fresh grammar per beam, replaying generated tokens.
# Backends don't support cloning grammar state, so
# replay is needed to reconstruct the FSM position.
grammar = backend.compile_grammar(request_type, grammar_spec)
assert beam.orig_prompt["type"] != "enc_dec"
prompt_len = len(beam.orig_prompt["prompt_token_ids"])
generated_tokens = beam.tokens[prompt_len:]
if generated_tokens:
grammar.accept_tokens("beam", generated_tokens)
if grammar.is_terminated():
result.append(None)
continue
grammar.fill_bitmask(bitmask, 0)
allowed_ids = _bitmask_to_token_ids(bitmask[0], vocab_size)
if not allowed_ids:
result.append(None)
continue
# The engine caps the size of allowed_token_ids. While the
# grammar still allows more tokens than the cap (e.g. inside
# free-form strings), skip the engine-side constraint and rely
# on the logprobs filtering in _beam_search_step instead.
beam_params = SamplingParams(
logprobs=base_params.logprobs,
max_tokens=1,
temperature=base_params.temperature,
allowed_token_ids=(
allowed_ids
if len(allowed_ids) <= _MAX_NUM_ALLOWED_TOKEN_IDS
else None
),
skip_clone=True,
)
result.append((beam_params, allowed_ids))
return result
+1
View File
@@ -1048,3 +1048,4 @@ class BeamSearchParams(
temperature: float = 0.0
length_penalty: float = 1.0
include_stop_str_in_output: bool = False
structured_outputs: StructuredOutputsParams | None = None