[Spec Decode] Remove Medusa and MLP speculator support (#61701)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
273a04964e
commit
8530aed2b7
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.models.interfaces import is_mixture_of_experts
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
|
||||
# Initialize logger
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MedusaProposer:
|
||||
"""Medusa proposer class for generating token sequences."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
# Save config parameters
|
||||
self.vllm_config = vllm_config
|
||||
assert vllm_config.speculative_config is not None, (
|
||||
"Speculative config must be set"
|
||||
)
|
||||
self.spec_config = vllm_config.speculative_config
|
||||
self.device = device
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.hidden_size = self.spec_config.draft_model_config.get_hidden_size()
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
self.num_speculative_tokens = self.spec_config.num_speculative_tokens
|
||||
|
||||
def propose(
|
||||
self,
|
||||
num_speculative_tokens: int,
|
||||
target_hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
slot_mappings: dict[str, torch.Tensor]
|
||||
| list[dict[str, torch.Tensor]]
|
||||
| None = None, # unused
|
||||
) -> torch.Tensor:
|
||||
assert num_speculative_tokens == self.num_speculative_tokens
|
||||
# Generate blocks and compute logits
|
||||
blocks = self.model(target_hidden_states)
|
||||
logits = self.model.compute_logits(blocks)
|
||||
|
||||
# Compute argmax for each Medusa head and stack into a single tensor
|
||||
# Shape: [batch_size, num_heads]
|
||||
draft_tokens = torch.stack([logit.argmax(dim=-1) for logit in logits], dim=1)
|
||||
|
||||
return draft_tokens
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
with set_model_tag("medusa_head"):
|
||||
self.model = get_model(
|
||||
vllm_config=self.vllm_config,
|
||||
model_config=self.spec_config.draft_model_config,
|
||||
)
|
||||
assert not (
|
||||
is_mixture_of_experts(self.model)
|
||||
and self.vllm_config.parallel_config.enable_eplb
|
||||
), "EPLB for Medusa is not supported"
|
||||
|
||||
@torch.inference_mode()
|
||||
def dummy_run(self, num_tokens: int) -> None:
|
||||
hidden_states = torch.zeros(
|
||||
(self.max_num_tokens, self.hidden_size),
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
with set_forward_context(None, self.vllm_config, num_tokens=num_tokens):
|
||||
self.model(hidden_states)
|
||||
@@ -180,7 +180,6 @@ from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
|
||||
from vllm.v1.spec_decode.medusa import MedusaProposer
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.spec_decode.ngram_proposer_gpu import (
|
||||
NgramProposerGPU,
|
||||
@@ -595,7 +594,6 @@ class GPUModelRunner(
|
||||
| EagleProposer
|
||||
| DFlashProposer
|
||||
| DraftModelProposer
|
||||
| MedusaProposer
|
||||
| ExtractHiddenStatesProposer
|
||||
| Gemma4Proposer
|
||||
| Step3p5MTPProposer
|
||||
@@ -646,10 +644,6 @@ class GPUModelRunner(
|
||||
self.use_aux_hidden_state_outputs = (
|
||||
self.drafter.eagle3_use_aux_hidden_state
|
||||
)
|
||||
elif self.speculative_config.method == "medusa":
|
||||
self.drafter = MedusaProposer(
|
||||
vllm_config=self.vllm_config, device=self.device
|
||||
)
|
||||
elif self.speculative_config.method == "extract_hidden_states":
|
||||
self.drafter = ExtractHiddenStatesProposer(
|
||||
vllm_config=self.vllm_config, device=self.device
|
||||
@@ -4830,33 +4824,6 @@ class GPUModelRunner(
|
||||
sampled_token_ids,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.method == "medusa":
|
||||
assert isinstance(sampled_token_ids, list)
|
||||
assert isinstance(self.drafter, MedusaProposer)
|
||||
|
||||
if sample_hidden_states.shape[0] == len(sampled_token_ids):
|
||||
# The input to the target model does not include draft tokens.
|
||||
hidden_states = sample_hidden_states
|
||||
else:
|
||||
indices = []
|
||||
offset = 0
|
||||
assert spec_decode_metadata is not None, (
|
||||
"No spec decode metadata for medusa"
|
||||
)
|
||||
for num_draft, tokens in zip(
|
||||
spec_decode_metadata.num_draft_tokens, sampled_token_ids
|
||||
):
|
||||
indices.append(offset + len(tokens) - 1)
|
||||
offset += num_draft + 1
|
||||
indices = async_tensor_h2d(indices, device=self.device)
|
||||
hidden_states = sample_hidden_states[indices]
|
||||
|
||||
draft_token_ids = self.drafter.propose(
|
||||
num_speculative_tokens=num_spec_tokens_to_schedule,
|
||||
target_hidden_states=hidden_states,
|
||||
sampling_metadata=sampling_metadata,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.uses_extract_hidden_states():
|
||||
assert isinstance(self.drafter, ExtractHiddenStatesProposer)
|
||||
assert isinstance(sampled_token_ids, torch.Tensor), (
|
||||
|
||||
Reference in New Issue
Block a user