[Bugfix] Don't let a structured-output request sample from an unmasked row (#54442)
Signed-off-by: ArcheyChen <45146080+ArcheyChen@users.noreply.github.com> Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: GPT-5.6 <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
GPT-5.6
Claude Opus 5
Wei Zhao
Wentao Ye
parent
31ec1c3aa1
commit
34c458d27f
@@ -253,6 +253,7 @@ class RejectionSampler:
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
expanded_local_pos: torch.Tensor,
|
||||
seq_lens_upper_bound_np: np.ndarray,
|
||||
verify_draft_sampled: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
processed_logits = self.sampler.apply_sampling_params(
|
||||
logits,
|
||||
@@ -267,7 +268,7 @@ class RejectionSampler:
|
||||
sampled, num_sampled = rejection_sample(
|
||||
processed_logits,
|
||||
draft_logits,
|
||||
draft_sampled,
|
||||
verify_draft_sampled,
|
||||
cu_num_logits,
|
||||
pos,
|
||||
idx_mapping,
|
||||
@@ -294,7 +295,10 @@ class RejectionSampler:
|
||||
pos: torch.Tensor,
|
||||
max_chunk_logits: int,
|
||||
max_num_logprobs: int,
|
||||
verify_draft_sampled: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]:
|
||||
if verify_draft_sampled is None:
|
||||
verify_draft_sampled = draft_sampled
|
||||
cu_num_logits_np = input_batch.cu_num_logits_np
|
||||
use_processed_logits = self.sampler.logprobs_mode in PROCESSED_LOGPROBS_MODES
|
||||
num_reqs = input_batch.num_reqs
|
||||
@@ -330,6 +334,7 @@ class RejectionSampler:
|
||||
input_batch.expanded_idx_mapping[lo:hi],
|
||||
input_batch.expanded_local_pos[lo:hi],
|
||||
input_batch.seq_lens_cpu_upper_bound.numpy()[start:end],
|
||||
verify_draft_sampled[lo:hi],
|
||||
)
|
||||
chunk_logprobs = self._get_logprobs_tensors(
|
||||
sampled,
|
||||
@@ -368,6 +373,7 @@ class RejectionSampler:
|
||||
logits: torch.Tensor,
|
||||
input_batch: InputBatch,
|
||||
draft_logits: torch.Tensor | None = None,
|
||||
invalid_drafts: torch.Tensor | None = None,
|
||||
) -> SamplerOutput:
|
||||
# NOTE(woosuk): We intentionally compute num_nans before sampling to make clear
|
||||
# that num_nans is computed before applying penalties and temperature.
|
||||
@@ -382,6 +388,14 @@ class RejectionSampler:
|
||||
self.sampler.req_states.prefill_len.gpu,
|
||||
)
|
||||
|
||||
# Marking a draft invalid (`is_valid_draft = draft_sampled >= 0`) pins the
|
||||
# accepted length before it reaches a permissive bitmask row. Only
|
||||
# verification sees the copy: `apply_sampling_params` and watermarking
|
||||
# keep the real draft ids.
|
||||
verify_draft_sampled = draft_sampled
|
||||
if invalid_drafts is not None:
|
||||
verify_draft_sampled = draft_sampled.masked_fill(invalid_drafts, -1)
|
||||
|
||||
max_num_logprobs = self.sampler.sampling_states.max_num_logprobs(
|
||||
input_batch.idx_mapping_np
|
||||
)
|
||||
@@ -394,6 +408,7 @@ class RejectionSampler:
|
||||
pos,
|
||||
chunk_logit_limit,
|
||||
max_num_logprobs,
|
||||
verify_draft_sampled,
|
||||
)
|
||||
|
||||
num_sampled, num_rejected = get_num_sampled_and_rejected(
|
||||
|
||||
Reference in New Issue
Block a user