[Bugfix] Reject structured outputs for diffusion decoders with a clear error (#45468)
Signed-off-by: Wayne Chiu <waynehacking8@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
+16
-1
@@ -720,7 +720,9 @@ class SamplingParams(
|
||||
self._validate_logits_processors(model_config)
|
||||
self._validate_allowed_token_ids(tokenizer)
|
||||
self._validate_spec_decode(speculative_config)
|
||||
self._validate_structured_outputs(structured_outputs_config, tokenizer)
|
||||
self._validate_structured_outputs(
|
||||
model_config, structured_outputs_config, tokenizer
|
||||
)
|
||||
|
||||
def _validate_logprobs(self, model_config: ModelConfig) -> None:
|
||||
max_logprobs = model_config.max_logprobs
|
||||
@@ -853,12 +855,25 @@ class SamplingParams(
|
||||
|
||||
def _validate_structured_outputs(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
structured_outputs_config: StructuredOutputsConfig | None,
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> None:
|
||||
if structured_outputs_config is None or self.structured_outputs is None:
|
||||
return
|
||||
|
||||
if model_config.is_diffusion:
|
||||
# Diffusion LLMs denoise a whole canvas of tokens in parallel
|
||||
# rather than sampling left-to-right, which the grammar FSM
|
||||
# requires. Without this check, requests fail mid-generation
|
||||
# with an FSM rejection (HTTP 500). See issue #45436.
|
||||
raise ValueError(
|
||||
"Structured outputs are not yet supported for diffusion "
|
||||
"language models. Remove the structured output constraint "
|
||||
"(e.g. `response_format`, `structured_outputs`) from the "
|
||||
"request."
|
||||
)
|
||||
|
||||
if tokenizer is None:
|
||||
raise ValueError(
|
||||
"Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501
|
||||
|
||||
Reference in New Issue
Block a user