[Bugfix][Spec Decode] Wire draft_probs into probabilistic draft_model rejection (#40269)
This commit is contained in:
@@ -67,7 +67,7 @@ SpeculativeMethod = Literal[
|
||||
NgramGPUTypes,
|
||||
]
|
||||
RejectionSampleMethod = Literal["standard", "synthetic"]
|
||||
DraftSampleMethod = Literal["greedy", "gumbel"]
|
||||
DraftSampleMethod = Literal["greedy", "probabilistic"]
|
||||
|
||||
|
||||
@config
|
||||
@@ -256,10 +256,10 @@ class SpeculativeConfig:
|
||||
draft_sample_method: DraftSampleMethod = "greedy"
|
||||
"""How the draft model samples tokens. 'greedy' always picks the argmax
|
||||
token, and the draft probabilities are treated as one-hot during rejection
|
||||
sampling. 'gumbel' adds Gumbel noise for stochastic sampling, and the full
|
||||
draft logits are used for the probability ratio test during rejection
|
||||
sampling. This comes at the cost of additional GPU memory usage. This
|
||||
parameter currently only applies to Model Runner V2."""
|
||||
sampling. 'probabilistic' samples stochastically from the draft
|
||||
distribution and uses the full draft logits for the probability ratio test
|
||||
during rejection sampling. This comes at the cost of additional GPU memory
|
||||
usage."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
|
||||
@@ -225,6 +225,11 @@ class SpecDecodeBaseProposer:
|
||||
device=device,
|
||||
with_numpy=True,
|
||||
)
|
||||
self._enable_probabilistic_draft_probs = (
|
||||
self.speculative_config.rejection_sample_method == "standard"
|
||||
and self.speculative_config.draft_sample_method == "probabilistic"
|
||||
)
|
||||
self._last_draft_probs: torch.Tensor | None = None
|
||||
|
||||
self._slot_mapping_buffer = torch.zeros(
|
||||
self.max_positions,
|
||||
@@ -389,6 +394,30 @@ class SpecDecodeBaseProposer:
|
||||
return self.model.get_top_tokens(hidden_states)
|
||||
return self.model.compute_logits(hidden_states).argmax(dim=-1)
|
||||
|
||||
def _sample_from_logits(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if not self._enable_probabilistic_draft_probs:
|
||||
return logits.argmax(dim=-1), None
|
||||
if sampling_metadata.all_greedy:
|
||||
return logits.argmax(dim=-1), None
|
||||
return compute_probs_and_sample_next_token(logits, sampling_metadata)
|
||||
|
||||
def _sample_draft_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if not self._enable_probabilistic_draft_probs or sampling_metadata.all_greedy:
|
||||
return self._greedy_sample(hidden_states), None
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
return self._sample_from_logits(logits, sampling_metadata)
|
||||
|
||||
def take_last_draft_probs(self) -> torch.Tensor | None:
|
||||
return self._last_draft_probs
|
||||
|
||||
def propose(
|
||||
self,
|
||||
# [num_tokens]
|
||||
@@ -408,6 +437,7 @@ class SpecDecodeBaseProposer:
|
||||
| list[dict[str, torch.Tensor]]
|
||||
| None = None,
|
||||
) -> torch.Tensor:
|
||||
self._last_draft_probs = None
|
||||
batch_size = common_attn_metadata.batch_size()
|
||||
|
||||
if self.method in ("eagle3", "dflash"):
|
||||
@@ -469,7 +499,13 @@ class SpecDecodeBaseProposer:
|
||||
|
||||
# Early exit if there is only one draft token to be generated.
|
||||
if self.num_speculative_tokens == 1 or self.parallel_drafting:
|
||||
draft_token_ids = self._greedy_sample(sample_hidden_states)
|
||||
draft_token_ids, draft_probs = self._sample_draft_tokens(
|
||||
sample_hidden_states, sampling_metadata
|
||||
)
|
||||
if draft_probs is not None:
|
||||
self._last_draft_probs = draft_probs.view(
|
||||
-1, self.num_speculative_tokens, draft_probs.shape[-1]
|
||||
).contiguous()
|
||||
return draft_token_ids.view(-1, self.num_speculative_tokens)
|
||||
|
||||
if self.uses_mrope:
|
||||
@@ -484,7 +520,10 @@ class SpecDecodeBaseProposer:
|
||||
# (which read via _get_positions) use the correct values.
|
||||
self.positions[:batch_size] = positions
|
||||
|
||||
draft_token_ids = self._greedy_sample(sample_hidden_states)
|
||||
draft_token_ids, draft_probs = self._sample_draft_tokens(
|
||||
sample_hidden_states, sampling_metadata
|
||||
)
|
||||
draft_probs_list = None if draft_probs is None else [draft_probs]
|
||||
|
||||
if self.allowed_attn_types is not None:
|
||||
for group_md in per_group_attn_metadata:
|
||||
@@ -584,11 +623,18 @@ class SpecDecodeBaseProposer:
|
||||
last_hidden_states, hidden_states = ret_hidden_states
|
||||
|
||||
hidden_states = hidden_states[:batch_size]
|
||||
draft_token_ids = self._greedy_sample(last_hidden_states[:batch_size])
|
||||
draft_token_ids, draft_probs = self._sample_draft_tokens(
|
||||
last_hidden_states[:batch_size], sampling_metadata
|
||||
)
|
||||
if draft_probs is not None:
|
||||
assert draft_probs_list is not None
|
||||
draft_probs_list.append(draft_probs)
|
||||
draft_token_ids_list.append(draft_token_ids)
|
||||
|
||||
# [batch_size, num_speculative_tokens]
|
||||
draft_token_ids = torch.stack(draft_token_ids_list, dim=1)
|
||||
if draft_probs_list is not None:
|
||||
self._last_draft_probs = torch.stack(draft_probs_list, dim=1).contiguous()
|
||||
return draft_token_ids
|
||||
|
||||
def _update_positions_dependent_metadata(
|
||||
|
||||
@@ -106,7 +106,7 @@ class EagleSpeculator:
|
||||
)
|
||||
|
||||
self.draft_logits: torch.Tensor | None = None
|
||||
if self.speculative_config.draft_sample_method == "gumbel":
|
||||
if self.speculative_config.draft_sample_method == "probabilistic":
|
||||
self.draft_logits = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
|
||||
@@ -806,6 +806,8 @@ class GPUModelRunner(
|
||||
|
||||
# Cached outputs.
|
||||
self._draft_token_ids: list[list[int]] | torch.Tensor | None = None
|
||||
self._draft_probs: torch.Tensor | None = None
|
||||
self._draft_prob_req_ids: list[str] | None = None
|
||||
# N-gram GPU path: async D2H buffer/event for per-request valid draft counts.
|
||||
self._num_valid_draft_tokens: torch.Tensor | None = None
|
||||
self._num_valid_draft_tokens_cpu: torch.Tensor | None = None
|
||||
@@ -3433,9 +3435,10 @@ class GPUModelRunner(
|
||||
draft_token_ids_cpu, _ = self._get_draft_token_ids_cpu()
|
||||
self.input_batch.update_async_spec_token_ids(draft_token_ids_cpu)
|
||||
|
||||
draft_probs = self._get_spec_decode_draft_probs(spec_decode_metadata)
|
||||
sampler_output = self.rejection_sampler(
|
||||
spec_decode_metadata,
|
||||
None, # draft_probs
|
||||
draft_probs,
|
||||
logits,
|
||||
sampling_metadata,
|
||||
)
|
||||
@@ -4280,6 +4283,8 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
self._draft_token_ids = None
|
||||
self._draft_probs = None
|
||||
self._draft_prob_req_ids = None
|
||||
self._draft_token_req_ids = None
|
||||
self.valid_sampled_token_count_gpu = None
|
||||
self.input_batch.prev_sampled_token_ids = None
|
||||
@@ -4374,6 +4379,8 @@ class GPUModelRunner(
|
||||
self._draft_token_ids = torch.zeros(
|
||||
1, device=self.device, dtype=torch.int32
|
||||
).expand(len(self.input_batch.req_ids), self.num_spec_tokens)
|
||||
self._draft_probs = None
|
||||
self._draft_prob_req_ids = None
|
||||
self._copy_draft_token_ids_to_cpu(scheduler_output, zeros_only=True)
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: bookkeep"):
|
||||
@@ -4599,6 +4606,35 @@ class GPUModelRunner(
|
||||
sampled_count_event.synchronize()
|
||||
return counts_cpu[: prev_sampled_token_ids.shape[0]].tolist()
|
||||
|
||||
def _get_spec_decode_draft_probs(
|
||||
self, spec_decode_metadata: SpecDecodeMetadata
|
||||
) -> torch.Tensor | None:
|
||||
if self._draft_probs is None or self._draft_prob_req_ids is None:
|
||||
return None
|
||||
|
||||
row_by_req_id = {
|
||||
req_id: idx for idx, req_id in enumerate(self._draft_prob_req_ids)
|
||||
}
|
||||
draft_probs_rows: list[torch.Tensor] = []
|
||||
for req_id, num_draft in zip(
|
||||
self.input_batch.req_ids, spec_decode_metadata.num_draft_tokens
|
||||
):
|
||||
if num_draft == 0:
|
||||
continue
|
||||
row_idx = row_by_req_id.get(req_id)
|
||||
if row_idx is None:
|
||||
logger.warning(
|
||||
"Missing cached draft probabilities for request %s; "
|
||||
"falling back to legacy speculative rejection behavior.",
|
||||
req_id,
|
||||
)
|
||||
return None
|
||||
draft_probs_rows.append(self._draft_probs[row_idx, :num_draft])
|
||||
|
||||
if not draft_probs_rows:
|
||||
return None
|
||||
return torch.cat(draft_probs_rows, dim=0).contiguous()
|
||||
|
||||
def propose_draft_token_ids(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
@@ -4614,6 +4650,8 @@ class GPUModelRunner(
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
spec_config = self.speculative_config
|
||||
assert spec_config is not None
|
||||
self._draft_probs = None
|
||||
self._draft_prob_req_ids = None
|
||||
if spec_config.method == "ngram":
|
||||
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
|
||||
|
||||
@@ -4859,6 +4897,11 @@ class GPUModelRunner(
|
||||
num_rejected_tokens_gpu=num_rejected_tokens_gpu,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
if hasattr(self.drafter, "take_last_draft_probs"):
|
||||
draft_probs = self.drafter.take_last_draft_probs()
|
||||
if draft_probs is not None:
|
||||
self._draft_probs = draft_probs
|
||||
self._draft_prob_req_ids = self.input_batch.req_ids.copy()
|
||||
|
||||
return draft_token_ids
|
||||
|
||||
@@ -5806,10 +5849,18 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
num_tokens = sum(len(ids) for ids in draft_token_ids)
|
||||
# draft_probs = torch.randn(
|
||||
# num_tokens, logits.shape[-1], device=self.device,
|
||||
# dtype=logits.dtype)
|
||||
draft_probs = None
|
||||
if (
|
||||
self.speculative_config.rejection_sample_method == "standard"
|
||||
and self.speculative_config.draft_sample_method == "probabilistic"
|
||||
):
|
||||
draft_probs = torch.rand(
|
||||
num_tokens,
|
||||
logits.shape[-1],
|
||||
device=self.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
draft_probs = torch.softmax(draft_probs, dim=-1)
|
||||
logits = torch.randn(
|
||||
num_tokens + num_reqs,
|
||||
logits.shape[-1],
|
||||
|
||||
Reference in New Issue
Block a user