[Bugfix] Don't let a structured-output request sample from an unmasked row (#54442)

Signed-off-by: ArcheyChen <45146080+ArcheyChen@users.noreply.github.com>
Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Co-authored-by: GPT-5.6 <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Ausar
2026-10-01 02:11:44 +00:00
committed by GitHub
co-authored by GPT-5.6 Claude Opus 5 Wei Zhao Wentao Ye
parent 31ec1c3aa1
commit 34c458d27f
8 changed files with 142 additions and 3 deletions
+11 -1
View File
@@ -164,7 +164,10 @@ from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
from vllm.v1.worker.gpu.structured_outputs import (
StructuredOutputsWorker,
grammar_invalid_drafts,
)
from vllm.v1.worker.gpu.ubatch_utils import (
UBatchRunner,
UBatchState,
@@ -1576,6 +1579,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
sample_hidden_states = hidden_states[input_batch.logits_indices]
logits = self.model.compute_logits(sample_hidden_states)
invalid_drafts = None
# A diffusion prefill has no logit rows even when a bitmask row
# arrived for it.
if grammar_output is not None and logits.shape[0] > 0:
@@ -1587,6 +1591,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
grammar_output.structured_output_request_ids,
grammar_output.grammar_bitmask,
)
invalid_drafts = grammar_invalid_drafts(
input_batch,
grammar_output.structured_output_request_ids,
grammar_output.num_acceptable_drafts,
)
sampler_output: SamplerOutput | None
if input_batch.num_reqs == 0:
@@ -1605,6 +1614,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch,
# Draft logits are needed for probabilistic rejection sampling.
self.speculator.draft_logits,
invalid_drafts,
)
if shard_metadata is not None: