[V1][Spec Decode] Always use argmax for sampling draft tokens (#16899)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon
2025-04-23 14:57:43 -07:00
committed by GitHub
parent 32d4b669d0
commit 41fb013d29
3 changed files with 18 additions and 23 deletions
+1 -4
View File
@@ -1230,7 +1230,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
target_hidden_states = hidden_states[token_indices]
target_slot_mapping = attn_metadata.slot_mapping[token_indices]
draft_token_ids, draft_probs = self.drafter.propose(
draft_token_ids = self.drafter.propose(
target_token_ids=target_token_ids,
target_positions=target_positions,
target_hidden_states=target_hidden_states,
@@ -1241,9 +1241,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
sampling_metadata=sampling_metadata,
)
spec_token_ids = draft_token_ids.tolist()
# TODO(woosuk): Cache draft_probs and use it for rejection sampling
# in the next step.
del draft_probs
# Clear KVConnector state after all KVs are generated.
if has_kv_transfer_group():