[Spec Decode] Remove Medusa and MLP speculator support (#61701)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
273a04964e
commit
8530aed2b7
@@ -180,7 +180,6 @@ from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
|
||||
from vllm.v1.spec_decode.medusa import MedusaProposer
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.spec_decode.ngram_proposer_gpu import (
|
||||
NgramProposerGPU,
|
||||
@@ -595,7 +594,6 @@ class GPUModelRunner(
|
||||
| EagleProposer
|
||||
| DFlashProposer
|
||||
| DraftModelProposer
|
||||
| MedusaProposer
|
||||
| ExtractHiddenStatesProposer
|
||||
| Gemma4Proposer
|
||||
| Step3p5MTPProposer
|
||||
@@ -646,10 +644,6 @@ class GPUModelRunner(
|
||||
self.use_aux_hidden_state_outputs = (
|
||||
self.drafter.eagle3_use_aux_hidden_state
|
||||
)
|
||||
elif self.speculative_config.method == "medusa":
|
||||
self.drafter = MedusaProposer(
|
||||
vllm_config=self.vllm_config, device=self.device
|
||||
)
|
||||
elif self.speculative_config.method == "extract_hidden_states":
|
||||
self.drafter = ExtractHiddenStatesProposer(
|
||||
vllm_config=self.vllm_config, device=self.device
|
||||
@@ -4830,33 +4824,6 @@ class GPUModelRunner(
|
||||
sampled_token_ids,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.method == "medusa":
|
||||
assert isinstance(sampled_token_ids, list)
|
||||
assert isinstance(self.drafter, MedusaProposer)
|
||||
|
||||
if sample_hidden_states.shape[0] == len(sampled_token_ids):
|
||||
# The input to the target model does not include draft tokens.
|
||||
hidden_states = sample_hidden_states
|
||||
else:
|
||||
indices = []
|
||||
offset = 0
|
||||
assert spec_decode_metadata is not None, (
|
||||
"No spec decode metadata for medusa"
|
||||
)
|
||||
for num_draft, tokens in zip(
|
||||
spec_decode_metadata.num_draft_tokens, sampled_token_ids
|
||||
):
|
||||
indices.append(offset + len(tokens) - 1)
|
||||
offset += num_draft + 1
|
||||
indices = async_tensor_h2d(indices, device=self.device)
|
||||
hidden_states = sample_hidden_states[indices]
|
||||
|
||||
draft_token_ids = self.drafter.propose(
|
||||
num_speculative_tokens=num_spec_tokens_to_schedule,
|
||||
target_hidden_states=hidden_states,
|
||||
sampling_metadata=sampling_metadata,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.uses_extract_hidden_states():
|
||||
assert isinstance(self.drafter, ExtractHiddenStatesProposer)
|
||||
assert isinstance(sampled_token_ids, torch.Tensor), (
|
||||
|
||||
Reference in New Issue
Block a user