[Bugfix] Don't let a structured-output request sample from an unmasked row (#54442)
Signed-off-by: ArcheyChen <45146080+ArcheyChen@users.noreply.github.com> Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: GPT-5.6 <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
GPT-5.6
Claude Opus 5
Wei Zhao
Wentao Ye
parent
31ec1c3aa1
commit
34c458d27f
@@ -909,6 +909,7 @@ def test_verify_rejects_unproposed_drafts():
|
||||
inputs["expanded_idx_mapping"],
|
||||
inputs["expanded_local_pos"],
|
||||
None,
|
||||
draft_sampled,
|
||||
)
|
||||
return sampled, num_sampled
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Which speculative drafts a structured-output request may accept.
|
||||
|
||||
`StructuredOutputManager.grammar_bitmask` fills row i before it inspects
|
||||
`req_tokens[i]`, so with the first -1 placeholder at index j rows 0..j carry a
|
||||
real mask and rows j+1.. carry the all-permissive `_full_mask`. Drafts j..K-1
|
||||
must therefore be rejected, or the request samples with no grammar constraint
|
||||
at a position the model really does sample.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.v1.worker.gpu.structured_outputs import grammar_invalid_drafts
|
||||
|
||||
|
||||
def _input_batch(req_ids, num_drafts, num_admitted=None):
|
||||
"""`num_admitted` is what adaptive verification kept on device."""
|
||||
num_admitted = num_drafts if num_admitted is None else num_admitted
|
||||
rows = [n + 1 for n in num_admitted]
|
||||
return SimpleNamespace(
|
||||
req_ids=list(req_ids),
|
||||
num_reqs=len(req_ids),
|
||||
num_draft_tokens=sum(num_drafts),
|
||||
num_draft_tokens_per_req=np.array(num_drafts, dtype=np.int32),
|
||||
cu_num_logits=torch.tensor([0, *np.cumsum(rows)], dtype=torch.int32),
|
||||
expanded_local_pos=torch.cat(
|
||||
[torch.arange(n, dtype=torch.int32) for n in rows]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _invalid_rows(batch, grammar_req_ids, num_acceptable):
|
||||
mask = grammar_invalid_drafts(batch, grammar_req_ids, num_acceptable)
|
||||
return None if mask is None else mask.nonzero().flatten().tolist()
|
||||
|
||||
|
||||
def test_reject_whole_window_when_nothing_was_backfilled():
|
||||
# Request "g" is at rows 0..3 (1 + 3 drafts), so its drafts are rows 1..3.
|
||||
batch = _input_batch(["g", "p"], [3, 3])
|
||||
assert _invalid_rows(batch, ["g"], [0]) == [1, 2, 3]
|
||||
|
||||
|
||||
def test_keep_the_drafts_the_bitmask_could_see():
|
||||
batch = _input_batch(["g", "p"], [3, 3])
|
||||
assert _invalid_rows(batch, ["g"], [2]) == [3]
|
||||
|
||||
|
||||
def test_follow_the_device_layout_under_adaptive_verification():
|
||||
# Scheduled 3 drafts each, but adaptive verification admitted 2 for "a" and
|
||||
# 3 for "b": the real rows are a=0..2, b=3..6, not the scheduled 0..3, 4..7.
|
||||
batch = _input_batch(["a", "b"], [3, 3], num_admitted=[2, 3])
|
||||
assert _invalid_rows(batch, ["a", "b"], [1, 0]) == [2, 4, 5, 6]
|
||||
Reference in New Issue
Block a user