[Bugfix][Spec Decode] Wire draft_probs into probabilistic draft_model rejection (#40269)
This commit is contained in:
@@ -806,6 +806,8 @@ class GPUModelRunner(
|
||||
|
||||
# Cached outputs.
|
||||
self._draft_token_ids: list[list[int]] | torch.Tensor | None = None
|
||||
self._draft_probs: torch.Tensor | None = None
|
||||
self._draft_prob_req_ids: list[str] | None = None
|
||||
# N-gram GPU path: async D2H buffer/event for per-request valid draft counts.
|
||||
self._num_valid_draft_tokens: torch.Tensor | None = None
|
||||
self._num_valid_draft_tokens_cpu: torch.Tensor | None = None
|
||||
@@ -3433,9 +3435,10 @@ class GPUModelRunner(
|
||||
draft_token_ids_cpu, _ = self._get_draft_token_ids_cpu()
|
||||
self.input_batch.update_async_spec_token_ids(draft_token_ids_cpu)
|
||||
|
||||
draft_probs = self._get_spec_decode_draft_probs(spec_decode_metadata)
|
||||
sampler_output = self.rejection_sampler(
|
||||
spec_decode_metadata,
|
||||
None, # draft_probs
|
||||
draft_probs,
|
||||
logits,
|
||||
sampling_metadata,
|
||||
)
|
||||
@@ -4280,6 +4283,8 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
self._draft_token_ids = None
|
||||
self._draft_probs = None
|
||||
self._draft_prob_req_ids = None
|
||||
self._draft_token_req_ids = None
|
||||
self.valid_sampled_token_count_gpu = None
|
||||
self.input_batch.prev_sampled_token_ids = None
|
||||
@@ -4374,6 +4379,8 @@ class GPUModelRunner(
|
||||
self._draft_token_ids = torch.zeros(
|
||||
1, device=self.device, dtype=torch.int32
|
||||
).expand(len(self.input_batch.req_ids), self.num_spec_tokens)
|
||||
self._draft_probs = None
|
||||
self._draft_prob_req_ids = None
|
||||
self._copy_draft_token_ids_to_cpu(scheduler_output, zeros_only=True)
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: bookkeep"):
|
||||
@@ -4599,6 +4606,35 @@ class GPUModelRunner(
|
||||
sampled_count_event.synchronize()
|
||||
return counts_cpu[: prev_sampled_token_ids.shape[0]].tolist()
|
||||
|
||||
def _get_spec_decode_draft_probs(
|
||||
self, spec_decode_metadata: SpecDecodeMetadata
|
||||
) -> torch.Tensor | None:
|
||||
if self._draft_probs is None or self._draft_prob_req_ids is None:
|
||||
return None
|
||||
|
||||
row_by_req_id = {
|
||||
req_id: idx for idx, req_id in enumerate(self._draft_prob_req_ids)
|
||||
}
|
||||
draft_probs_rows: list[torch.Tensor] = []
|
||||
for req_id, num_draft in zip(
|
||||
self.input_batch.req_ids, spec_decode_metadata.num_draft_tokens
|
||||
):
|
||||
if num_draft == 0:
|
||||
continue
|
||||
row_idx = row_by_req_id.get(req_id)
|
||||
if row_idx is None:
|
||||
logger.warning(
|
||||
"Missing cached draft probabilities for request %s; "
|
||||
"falling back to legacy speculative rejection behavior.",
|
||||
req_id,
|
||||
)
|
||||
return None
|
||||
draft_probs_rows.append(self._draft_probs[row_idx, :num_draft])
|
||||
|
||||
if not draft_probs_rows:
|
||||
return None
|
||||
return torch.cat(draft_probs_rows, dim=0).contiguous()
|
||||
|
||||
def propose_draft_token_ids(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
@@ -4614,6 +4650,8 @@ class GPUModelRunner(
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
spec_config = self.speculative_config
|
||||
assert spec_config is not None
|
||||
self._draft_probs = None
|
||||
self._draft_prob_req_ids = None
|
||||
if spec_config.method == "ngram":
|
||||
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
|
||||
|
||||
@@ -4859,6 +4897,11 @@ class GPUModelRunner(
|
||||
num_rejected_tokens_gpu=num_rejected_tokens_gpu,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
if hasattr(self.drafter, "take_last_draft_probs"):
|
||||
draft_probs = self.drafter.take_last_draft_probs()
|
||||
if draft_probs is not None:
|
||||
self._draft_probs = draft_probs
|
||||
self._draft_prob_req_ids = self.input_batch.req_ids.copy()
|
||||
|
||||
return draft_token_ids
|
||||
|
||||
@@ -5806,10 +5849,18 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
num_tokens = sum(len(ids) for ids in draft_token_ids)
|
||||
# draft_probs = torch.randn(
|
||||
# num_tokens, logits.shape[-1], device=self.device,
|
||||
# dtype=logits.dtype)
|
||||
draft_probs = None
|
||||
if (
|
||||
self.speculative_config.rejection_sample_method == "standard"
|
||||
and self.speculative_config.draft_sample_method == "probabilistic"
|
||||
):
|
||||
draft_probs = torch.rand(
|
||||
num_tokens,
|
||||
logits.shape[-1],
|
||||
device=self.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
draft_probs = torch.softmax(draft_probs, dim=-1)
|
||||
logits = torch.randn(
|
||||
num_tokens + num_reqs,
|
||||
logits.shape[-1],
|
||||
|
||||
Reference in New Issue
Block a user