[Bugfix][Spec Decode] Wire draft_probs into probabilistic draft_model rejection (#40269)

This commit is contained in:
Siddharth Bedekar
2026-05-13 21:04:03 -04:00
committed by GitHub
parent 665f9c4253
commit f51f6844f9
7 changed files with 264 additions and 13 deletions
+5 -5
View File
@@ -67,7 +67,7 @@ SpeculativeMethod = Literal[
NgramGPUTypes,
]
RejectionSampleMethod = Literal["standard", "synthetic"]
DraftSampleMethod = Literal["greedy", "gumbel"]
DraftSampleMethod = Literal["greedy", "probabilistic"]
@config
@@ -256,10 +256,10 @@ class SpeculativeConfig:
draft_sample_method: DraftSampleMethod = "greedy"
"""How the draft model samples tokens. 'greedy' always picks the argmax
token, and the draft probabilities are treated as one-hot during rejection
sampling. 'gumbel' adds Gumbel noise for stochastic sampling, and the full
draft logits are used for the probability ratio test during rejection
sampling. This comes at the cost of additional GPU memory usage. This
parameter currently only applies to Model Runner V2."""
sampling. 'probabilistic' samples stochastically from the draft
distribution and uses the full draft logits for the probability ratio test
during rejection sampling. This comes at the cost of additional GPU memory
usage."""
def compute_hash(self) -> str:
"""
+49 -3
View File
@@ -225,6 +225,11 @@ class SpecDecodeBaseProposer:
device=device,
with_numpy=True,
)
self._enable_probabilistic_draft_probs = (
self.speculative_config.rejection_sample_method == "standard"
and self.speculative_config.draft_sample_method == "probabilistic"
)
self._last_draft_probs: torch.Tensor | None = None
self._slot_mapping_buffer = torch.zeros(
self.max_positions,
@@ -389,6 +394,30 @@ class SpecDecodeBaseProposer:
return self.model.get_top_tokens(hidden_states)
return self.model.compute_logits(hidden_states).argmax(dim=-1)
def _sample_from_logits(
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if not self._enable_probabilistic_draft_probs:
return logits.argmax(dim=-1), None
if sampling_metadata.all_greedy:
return logits.argmax(dim=-1), None
return compute_probs_and_sample_next_token(logits, sampling_metadata)
def _sample_draft_tokens(
self,
hidden_states: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if not self._enable_probabilistic_draft_probs or sampling_metadata.all_greedy:
return self._greedy_sample(hidden_states), None
logits = self.model.compute_logits(hidden_states)
return self._sample_from_logits(logits, sampling_metadata)
def take_last_draft_probs(self) -> torch.Tensor | None:
return self._last_draft_probs
def propose(
self,
# [num_tokens]
@@ -408,6 +437,7 @@ class SpecDecodeBaseProposer:
| list[dict[str, torch.Tensor]]
| None = None,
) -> torch.Tensor:
self._last_draft_probs = None
batch_size = common_attn_metadata.batch_size()
if self.method in ("eagle3", "dflash"):
@@ -469,7 +499,13 @@ class SpecDecodeBaseProposer:
# Early exit if there is only one draft token to be generated.
if self.num_speculative_tokens == 1 or self.parallel_drafting:
draft_token_ids = self._greedy_sample(sample_hidden_states)
draft_token_ids, draft_probs = self._sample_draft_tokens(
sample_hidden_states, sampling_metadata
)
if draft_probs is not None:
self._last_draft_probs = draft_probs.view(
-1, self.num_speculative_tokens, draft_probs.shape[-1]
).contiguous()
return draft_token_ids.view(-1, self.num_speculative_tokens)
if self.uses_mrope:
@@ -484,7 +520,10 @@ class SpecDecodeBaseProposer:
# (which read via _get_positions) use the correct values.
self.positions[:batch_size] = positions
draft_token_ids = self._greedy_sample(sample_hidden_states)
draft_token_ids, draft_probs = self._sample_draft_tokens(
sample_hidden_states, sampling_metadata
)
draft_probs_list = None if draft_probs is None else [draft_probs]
if self.allowed_attn_types is not None:
for group_md in per_group_attn_metadata:
@@ -584,11 +623,18 @@ class SpecDecodeBaseProposer:
last_hidden_states, hidden_states = ret_hidden_states
hidden_states = hidden_states[:batch_size]
draft_token_ids = self._greedy_sample(last_hidden_states[:batch_size])
draft_token_ids, draft_probs = self._sample_draft_tokens(
last_hidden_states[:batch_size], sampling_metadata
)
if draft_probs is not None:
assert draft_probs_list is not None
draft_probs_list.append(draft_probs)
draft_token_ids_list.append(draft_token_ids)
# [batch_size, num_speculative_tokens]
draft_token_ids = torch.stack(draft_token_ids_list, dim=1)
if draft_probs_list is not None:
self._last_draft_probs = torch.stack(draft_probs_list, dim=1).contiguous()
return draft_token_ids
def _update_positions_dependent_metadata(
@@ -106,7 +106,7 @@ class EagleSpeculator:
)
self.draft_logits: torch.Tensor | None = None
if self.speculative_config.draft_sample_method == "gumbel":
if self.speculative_config.draft_sample_method == "probabilistic":
self.draft_logits = torch.zeros(
self.max_num_reqs,
self.num_speculative_steps,
+55 -4
View File
@@ -806,6 +806,8 @@ class GPUModelRunner(
# Cached outputs.
self._draft_token_ids: list[list[int]] | torch.Tensor | None = None
self._draft_probs: torch.Tensor | None = None
self._draft_prob_req_ids: list[str] | None = None
# N-gram GPU path: async D2H buffer/event for per-request valid draft counts.
self._num_valid_draft_tokens: torch.Tensor | None = None
self._num_valid_draft_tokens_cpu: torch.Tensor | None = None
@@ -3433,9 +3435,10 @@ class GPUModelRunner(
draft_token_ids_cpu, _ = self._get_draft_token_ids_cpu()
self.input_batch.update_async_spec_token_ids(draft_token_ids_cpu)
draft_probs = self._get_spec_decode_draft_probs(spec_decode_metadata)
sampler_output = self.rejection_sampler(
spec_decode_metadata,
None, # draft_probs
draft_probs,
logits,
sampling_metadata,
)
@@ -4280,6 +4283,8 @@ class GPUModelRunner(
)
self._draft_token_ids = None
self._draft_probs = None
self._draft_prob_req_ids = None
self._draft_token_req_ids = None
self.valid_sampled_token_count_gpu = None
self.input_batch.prev_sampled_token_ids = None
@@ -4374,6 +4379,8 @@ class GPUModelRunner(
self._draft_token_ids = torch.zeros(
1, device=self.device, dtype=torch.int32
).expand(len(self.input_batch.req_ids), self.num_spec_tokens)
self._draft_probs = None
self._draft_prob_req_ids = None
self._copy_draft_token_ids_to_cpu(scheduler_output, zeros_only=True)
with record_function_or_nullcontext("gpu_model_runner: bookkeep"):
@@ -4599,6 +4606,35 @@ class GPUModelRunner(
sampled_count_event.synchronize()
return counts_cpu[: prev_sampled_token_ids.shape[0]].tolist()
def _get_spec_decode_draft_probs(
self, spec_decode_metadata: SpecDecodeMetadata
) -> torch.Tensor | None:
if self._draft_probs is None or self._draft_prob_req_ids is None:
return None
row_by_req_id = {
req_id: idx for idx, req_id in enumerate(self._draft_prob_req_ids)
}
draft_probs_rows: list[torch.Tensor] = []
for req_id, num_draft in zip(
self.input_batch.req_ids, spec_decode_metadata.num_draft_tokens
):
if num_draft == 0:
continue
row_idx = row_by_req_id.get(req_id)
if row_idx is None:
logger.warning(
"Missing cached draft probabilities for request %s; "
"falling back to legacy speculative rejection behavior.",
req_id,
)
return None
draft_probs_rows.append(self._draft_probs[row_idx, :num_draft])
if not draft_probs_rows:
return None
return torch.cat(draft_probs_rows, dim=0).contiguous()
def propose_draft_token_ids(
self,
scheduler_output: "SchedulerOutput",
@@ -4614,6 +4650,8 @@ class GPUModelRunner(
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
spec_config = self.speculative_config
assert spec_config is not None
self._draft_probs = None
self._draft_prob_req_ids = None
if spec_config.method == "ngram":
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
@@ -4859,6 +4897,11 @@ class GPUModelRunner(
num_rejected_tokens_gpu=num_rejected_tokens_gpu,
slot_mappings=slot_mappings,
)
if hasattr(self.drafter, "take_last_draft_probs"):
draft_probs = self.drafter.take_last_draft_probs()
if draft_probs is not None:
self._draft_probs = draft_probs
self._draft_prob_req_ids = self.input_batch.req_ids.copy()
return draft_token_ids
@@ -5806,10 +5849,18 @@ class GPUModelRunner(
)
num_tokens = sum(len(ids) for ids in draft_token_ids)
# draft_probs = torch.randn(
# num_tokens, logits.shape[-1], device=self.device,
# dtype=logits.dtype)
draft_probs = None
if (
self.speculative_config.rejection_sample_method == "standard"
and self.speculative_config.draft_sample_method == "probabilistic"
):
draft_probs = torch.rand(
num_tokens,
logits.shape[-1],
device=self.device,
dtype=torch.float32,
)
draft_probs = torch.softmax(draft_probs, dim=-1)
logits = torch.randn(
num_tokens + num_reqs,
logits.shape[-1],