[Bugfix] Reject structured outputs for diffusion decoders with a clear error (#45468)
Signed-off-by: Wayne Chiu <waynehacking8@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,50 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Request-time validation of structured output requests."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.config import StructuredOutputsConfig
|
||||||
|
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.cpu_test
|
||||||
|
|
||||||
|
JSON_SCHEMA = {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"invoice_id": {"type": "string"},
|
||||||
|
"customer": {"type": "string"},
|
||||||
|
},
|
||||||
|
"required": ["invoice_id", "customer"],
|
||||||
|
"additionalProperties": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class _StubModelConfig:
|
||||||
|
def __init__(self, is_diffusion: bool):
|
||||||
|
self.is_diffusion = is_diffusion
|
||||||
|
|
||||||
|
|
||||||
|
def test_structured_outputs_rejected_for_diffusion_models():
|
||||||
|
"""Diffusion LLMs denoise the canvas in parallel, which is incompatible
|
||||||
|
with the token-by-token grammar FSM. The request must fail with a clear
|
||||||
|
validation error instead of an FSM rejection mid-generation (#45436)."""
|
||||||
|
params = SamplingParams(
|
||||||
|
structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="not yet supported for diffusion"):
|
||||||
|
params._validate_structured_outputs(
|
||||||
|
_StubModelConfig(is_diffusion=True),
|
||||||
|
StructuredOutputsConfig(),
|
||||||
|
tokenizer=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_plain_request_allowed_for_diffusion_models():
|
||||||
|
"""Requests without structured outputs are unaffected by the guard."""
|
||||||
|
params = SamplingParams()
|
||||||
|
params._validate_structured_outputs(
|
||||||
|
_StubModelConfig(is_diffusion=True),
|
||||||
|
StructuredOutputsConfig(),
|
||||||
|
tokenizer=None,
|
||||||
|
)
|
||||||
+16
-1
@@ -720,7 +720,9 @@ class SamplingParams(
|
|||||||
self._validate_logits_processors(model_config)
|
self._validate_logits_processors(model_config)
|
||||||
self._validate_allowed_token_ids(tokenizer)
|
self._validate_allowed_token_ids(tokenizer)
|
||||||
self._validate_spec_decode(speculative_config)
|
self._validate_spec_decode(speculative_config)
|
||||||
self._validate_structured_outputs(structured_outputs_config, tokenizer)
|
self._validate_structured_outputs(
|
||||||
|
model_config, structured_outputs_config, tokenizer
|
||||||
|
)
|
||||||
|
|
||||||
def _validate_logprobs(self, model_config: ModelConfig) -> None:
|
def _validate_logprobs(self, model_config: ModelConfig) -> None:
|
||||||
max_logprobs = model_config.max_logprobs
|
max_logprobs = model_config.max_logprobs
|
||||||
@@ -853,12 +855,25 @@ class SamplingParams(
|
|||||||
|
|
||||||
def _validate_structured_outputs(
|
def _validate_structured_outputs(
|
||||||
self,
|
self,
|
||||||
|
model_config: ModelConfig,
|
||||||
structured_outputs_config: StructuredOutputsConfig | None,
|
structured_outputs_config: StructuredOutputsConfig | None,
|
||||||
tokenizer: TokenizerLike | None,
|
tokenizer: TokenizerLike | None,
|
||||||
) -> None:
|
) -> None:
|
||||||
if structured_outputs_config is None or self.structured_outputs is None:
|
if structured_outputs_config is None or self.structured_outputs is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if model_config.is_diffusion:
|
||||||
|
# Diffusion LLMs denoise a whole canvas of tokens in parallel
|
||||||
|
# rather than sampling left-to-right, which the grammar FSM
|
||||||
|
# requires. Without this check, requests fail mid-generation
|
||||||
|
# with an FSM rejection (HTTP 500). See issue #45436.
|
||||||
|
raise ValueError(
|
||||||
|
"Structured outputs are not yet supported for diffusion "
|
||||||
|
"language models. Remove the structured output constraint "
|
||||||
|
"(e.g. `response_format`, `structured_outputs`) from the "
|
||||||
|
"request."
|
||||||
|
)
|
||||||
|
|
||||||
if tokenizer is None:
|
if tokenizer is None:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501
|
"Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501
|
||||||
|
|||||||
Reference in New Issue
Block a user