[Bugfix] Don't let a structured-output request sample from an unmasked row (#54442)
Signed-off-by: ArcheyChen <45146080+ArcheyChen@users.noreply.github.com> Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: GPT-5.6 <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
GPT-5.6
Claude Opus 5
Wei Zhao
Wentao Ye
parent
31ec1c3aa1
commit
34c458d27f
@@ -164,7 +164,10 @@ from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
|
||||
from vllm.v1.worker.gpu.structured_outputs import (
|
||||
StructuredOutputsWorker,
|
||||
grammar_invalid_drafts,
|
||||
)
|
||||
from vllm.v1.worker.gpu.ubatch_utils import (
|
||||
UBatchRunner,
|
||||
UBatchState,
|
||||
@@ -1576,6 +1579,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
sample_hidden_states = hidden_states[input_batch.logits_indices]
|
||||
logits = self.model.compute_logits(sample_hidden_states)
|
||||
|
||||
invalid_drafts = None
|
||||
# A diffusion prefill has no logit rows even when a bitmask row
|
||||
# arrived for it.
|
||||
if grammar_output is not None and logits.shape[0] > 0:
|
||||
@@ -1587,6 +1591,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
grammar_output.structured_output_request_ids,
|
||||
grammar_output.grammar_bitmask,
|
||||
)
|
||||
invalid_drafts = grammar_invalid_drafts(
|
||||
input_batch,
|
||||
grammar_output.structured_output_request_ids,
|
||||
grammar_output.num_acceptable_drafts,
|
||||
)
|
||||
|
||||
sampler_output: SamplerOutput | None
|
||||
if input_batch.num_reqs == 0:
|
||||
@@ -1605,6 +1614,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch,
|
||||
# Draft logits are needed for probabilistic rejection sampling.
|
||||
self.speculator.draft_logits,
|
||||
invalid_drafts,
|
||||
)
|
||||
|
||||
if shard_metadata is not None:
|
||||
|
||||
Reference in New Issue
Block a user