[Bugfix][Spec Decode] Wire draft_probs into probabilistic draft_model rejection (#40269)

This commit is contained in:
Siddharth Bedekar
2026-05-13 21:04:03 -04:00
committed by GitHub
parent 665f9c4253
commit f51f6844f9
7 changed files with 264 additions and 13 deletions
+55 -4
View File
@@ -806,6 +806,8 @@ class GPUModelRunner(
# Cached outputs.
self._draft_token_ids: list[list[int]] | torch.Tensor | None = None
self._draft_probs: torch.Tensor | None = None
self._draft_prob_req_ids: list[str] | None = None
# N-gram GPU path: async D2H buffer/event for per-request valid draft counts.
self._num_valid_draft_tokens: torch.Tensor | None = None
self._num_valid_draft_tokens_cpu: torch.Tensor | None = None
@@ -3433,9 +3435,10 @@ class GPUModelRunner(
draft_token_ids_cpu, _ = self._get_draft_token_ids_cpu()
self.input_batch.update_async_spec_token_ids(draft_token_ids_cpu)
draft_probs = self._get_spec_decode_draft_probs(spec_decode_metadata)
sampler_output = self.rejection_sampler(
spec_decode_metadata,
None, # draft_probs
draft_probs,
logits,
sampling_metadata,
)
@@ -4280,6 +4283,8 @@ class GPUModelRunner(
)
self._draft_token_ids = None
self._draft_probs = None
self._draft_prob_req_ids = None
self._draft_token_req_ids = None
self.valid_sampled_token_count_gpu = None
self.input_batch.prev_sampled_token_ids = None
@@ -4374,6 +4379,8 @@ class GPUModelRunner(
self._draft_token_ids = torch.zeros(
1, device=self.device, dtype=torch.int32
).expand(len(self.input_batch.req_ids), self.num_spec_tokens)
self._draft_probs = None
self._draft_prob_req_ids = None
self._copy_draft_token_ids_to_cpu(scheduler_output, zeros_only=True)
with record_function_or_nullcontext("gpu_model_runner: bookkeep"):
@@ -4599,6 +4606,35 @@ class GPUModelRunner(
sampled_count_event.synchronize()
return counts_cpu[: prev_sampled_token_ids.shape[0]].tolist()
def _get_spec_decode_draft_probs(
self, spec_decode_metadata: SpecDecodeMetadata
) -> torch.Tensor | None:
if self._draft_probs is None or self._draft_prob_req_ids is None:
return None
row_by_req_id = {
req_id: idx for idx, req_id in enumerate(self._draft_prob_req_ids)
}
draft_probs_rows: list[torch.Tensor] = []
for req_id, num_draft in zip(
self.input_batch.req_ids, spec_decode_metadata.num_draft_tokens
):
if num_draft == 0:
continue
row_idx = row_by_req_id.get(req_id)
if row_idx is None:
logger.warning(
"Missing cached draft probabilities for request %s; "
"falling back to legacy speculative rejection behavior.",
req_id,
)
return None
draft_probs_rows.append(self._draft_probs[row_idx, :num_draft])
if not draft_probs_rows:
return None
return torch.cat(draft_probs_rows, dim=0).contiguous()
def propose_draft_token_ids(
self,
scheduler_output: "SchedulerOutput",
@@ -4614,6 +4650,8 @@ class GPUModelRunner(
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
spec_config = self.speculative_config
assert spec_config is not None
self._draft_probs = None
self._draft_prob_req_ids = None
if spec_config.method == "ngram":
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
@@ -4859,6 +4897,11 @@ class GPUModelRunner(
num_rejected_tokens_gpu=num_rejected_tokens_gpu,
slot_mappings=slot_mappings,
)
if hasattr(self.drafter, "take_last_draft_probs"):
draft_probs = self.drafter.take_last_draft_probs()
if draft_probs is not None:
self._draft_probs = draft_probs
self._draft_prob_req_ids = self.input_batch.req_ids.copy()
return draft_token_ids
@@ -5806,10 +5849,18 @@ class GPUModelRunner(
)
num_tokens = sum(len(ids) for ids in draft_token_ids)
# draft_probs = torch.randn(
# num_tokens, logits.shape[-1], device=self.device,
# dtype=logits.dtype)
draft_probs = None
if (
self.speculative_config.rejection_sample_method == "standard"
and self.speculative_config.draft_sample_method == "probabilistic"
):
draft_probs = torch.rand(
num_tokens,
logits.shape[-1],
device=self.device,
dtype=torch.float32,
)
draft_probs = torch.softmax(draft_probs, dim=-1)
logits = torch.randn(
num_tokens + num_reqs,
logits.shape[-1],