[V1][Spec Decode] Always use argmax for sampling draft tokens (#16899)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
@@ -51,7 +51,7 @@ class EagleProposer:
|
||||
# [batch_size, max_num_blocks_per_req]
|
||||
block_table: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
num_tokens = target_token_ids.shape[0]
|
||||
batch_size = next_token_ids.shape[0]
|
||||
last_token_indices = cu_num_tokens[1:] - 1
|
||||
@@ -94,17 +94,15 @@ class EagleProposer:
|
||||
)
|
||||
sample_hidden_states = hidden_states[last_token_indices]
|
||||
logits = self.model.compute_logits(sample_hidden_states, None)
|
||||
draft_token_ids, draft_probs = compute_probs_and_sample_next_token(
|
||||
logits, sampling_metadata)
|
||||
draft_token_ids = logits.argmax(dim=-1)
|
||||
|
||||
# Early exit if there is only one draft token to be generated.
|
||||
if self.num_speculative_tokens == 1:
|
||||
# [batch_size, 1] and [batch_size, 1, vocab_size]
|
||||
return draft_token_ids.view(-1, 1), draft_probs.unsqueeze(dim=1)
|
||||
# [batch_size, 1]
|
||||
return draft_token_ids.view(-1, 1)
|
||||
|
||||
# Generate the remaining draft tokens.
|
||||
draft_token_ids_list = [draft_token_ids]
|
||||
draft_probs_list = [draft_probs]
|
||||
|
||||
positions = target_positions[last_token_indices]
|
||||
hidden_states = sample_hidden_states
|
||||
@@ -159,16 +157,12 @@ class EagleProposer:
|
||||
positions=clamped_positions,
|
||||
)
|
||||
logits = self.model.compute_logits(hidden_states, None)
|
||||
draft_token_ids, probs = compute_probs_and_sample_next_token(
|
||||
logits, sampling_metadata)
|
||||
draft_token_ids = logits.argmax(dim=-1)
|
||||
draft_token_ids_list.append(draft_token_ids)
|
||||
draft_probs_list.append(probs)
|
||||
|
||||
# [batch_size, num_speculative_tokens]
|
||||
draft_token_ids = torch.stack(draft_token_ids_list, dim=1)
|
||||
# [batch_size, num_speculative_tokens, vocab_size]
|
||||
draft_probs = torch.stack(draft_probs_list, dim=1)
|
||||
return draft_token_ids, draft_probs
|
||||
return draft_token_ids
|
||||
|
||||
@staticmethod
|
||||
def prepare_inputs(
|
||||
@@ -238,6 +232,10 @@ class EagleProposer:
|
||||
self.model.lm_head = target_model.lm_head
|
||||
|
||||
|
||||
# NOTE(woosuk): Currently, the below code is not used and we always use argmax
|
||||
# to sample the draft tokens. We will use this after we find a way to manage
|
||||
# the draft prob tensor.
|
||||
# Refer to https://github.com/vllm-project/vllm/pull/16899 for the details.
|
||||
# FIXME(woosuk): The logic here is duplicated with the main sampling code.
|
||||
# We should refactor this to reuse the same sampling implementation.
|
||||
def compute_probs_and_sample_next_token(
|
||||
|
||||
Reference in New Issue
Block a user