[Bugfix] Don't let a structured-output request sample from an unmasked row (#54442)

Signed-off-by: ArcheyChen <45146080+ArcheyChen@users.noreply.github.com>
Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Co-authored-by: GPT-5.6 <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Ausar
2026-10-01 02:11:44 +00:00
committed by GitHub
co-authored by GPT-5.6 Claude Opus 5 Wei Zhao Wentao Ye
parent 31ec1c3aa1
commit 34c458d27f
8 changed files with 142 additions and 3 deletions
@@ -253,6 +253,7 @@ class RejectionSampler:
expanded_idx_mapping: torch.Tensor,
expanded_local_pos: torch.Tensor,
seq_lens_upper_bound_np: np.ndarray,
verify_draft_sampled: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
processed_logits = self.sampler.apply_sampling_params(
logits,
@@ -267,7 +268,7 @@ class RejectionSampler:
sampled, num_sampled = rejection_sample(
processed_logits,
draft_logits,
draft_sampled,
verify_draft_sampled,
cu_num_logits,
pos,
idx_mapping,
@@ -294,7 +295,10 @@ class RejectionSampler:
pos: torch.Tensor,
max_chunk_logits: int,
max_num_logprobs: int,
verify_draft_sampled: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]:
if verify_draft_sampled is None:
verify_draft_sampled = draft_sampled
cu_num_logits_np = input_batch.cu_num_logits_np
use_processed_logits = self.sampler.logprobs_mode in PROCESSED_LOGPROBS_MODES
num_reqs = input_batch.num_reqs
@@ -330,6 +334,7 @@ class RejectionSampler:
input_batch.expanded_idx_mapping[lo:hi],
input_batch.expanded_local_pos[lo:hi],
input_batch.seq_lens_cpu_upper_bound.numpy()[start:end],
verify_draft_sampled[lo:hi],
)
chunk_logprobs = self._get_logprobs_tensors(
sampled,
@@ -368,6 +373,7 @@ class RejectionSampler:
logits: torch.Tensor,
input_batch: InputBatch,
draft_logits: torch.Tensor | None = None,
invalid_drafts: torch.Tensor | None = None,
) -> SamplerOutput:
# NOTE(woosuk): We intentionally compute num_nans before sampling to make clear
# that num_nans is computed before applying penalties and temperature.
@@ -382,6 +388,14 @@ class RejectionSampler:
self.sampler.req_states.prefill_len.gpu,
)
# Marking a draft invalid (`is_valid_draft = draft_sampled >= 0`) pins the
# accepted length before it reaches a permissive bitmask row. Only
# verification sees the copy: `apply_sampling_params` and watermarking
# keep the real draft ids.
verify_draft_sampled = draft_sampled
if invalid_drafts is not None:
verify_draft_sampled = draft_sampled.masked_fill(invalid_drafts, -1)
max_num_logprobs = self.sampler.sampling_states.max_num_logprobs(
input_batch.idx_mapping_np
)
@@ -394,6 +408,7 @@ class RejectionSampler:
pos,
chunk_logit_limit,
max_num_logprobs,
verify_draft_sampled,
)
num_sampled, num_rejected = get_num_sampled_and_rejected(