[Spec Decode] Remove Medusa and MLP speculator support (#61701)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
273a04964e
commit
8530aed2b7
@@ -72,8 +72,6 @@ EagleModelTypes = Literal[
|
||||
]
|
||||
SpeculativeMethod = Literal[
|
||||
"ngram",
|
||||
"medusa",
|
||||
"mlp_speculator",
|
||||
"draft_model",
|
||||
"suffix",
|
||||
"custom_class",
|
||||
@@ -1272,20 +1270,12 @@ class SpeculativeConfig:
|
||||
self.prompt_lookup_min = 0
|
||||
|
||||
if self.model is not None:
|
||||
# Old-format Medusa checkpoints (e.g. FasterDecoding/medusa-*)
|
||||
# lack a model_type key in config.json, so AutoConfig cannot
|
||||
# detect them. When the method is explicitly "medusa", inject
|
||||
# model_type so MedusaConfig.from_pretrained is used instead.
|
||||
draft_hf_overrides: HfOverrides
|
||||
if self.method == "medusa":
|
||||
draft_hf_overrides = {"model_type": "medusa"}
|
||||
else:
|
||||
# Compose any callable hf_overrides set on the target so the
|
||||
# draft config receives the same transform (e.g. the test
|
||||
# shrink). Dict overrides stay target-only.
|
||||
draft_hf_overrides = SpeculativeConfig.compose_draft_hf_overrides(
|
||||
self.target_model_config.hf_overrides
|
||||
)
|
||||
# Compose any callable hf_overrides set on the target so the
|
||||
# draft config receives the same transform (e.g. the test
|
||||
# shrink). Dict overrides stay target-only.
|
||||
draft_hf_overrides = SpeculativeConfig.compose_draft_hf_overrides(
|
||||
self.target_model_config.hf_overrides
|
||||
)
|
||||
self.draft_model_config = ModelConfig(
|
||||
model=self.model,
|
||||
runner="draft",
|
||||
@@ -1312,17 +1302,6 @@ class SpeculativeConfig:
|
||||
config_format=self.target_model_config.config_format,
|
||||
)
|
||||
|
||||
# Old-format Medusa checkpoints (e.g. FasterDecoding/medusa-*)
|
||||
# omit vocab_size in config.json, so MedusaConfig falls back to
|
||||
# its default (32001). Align with the target model's vocab size
|
||||
# to avoid shape mismatches when loading LM-head weights.
|
||||
if self.method == "medusa":
|
||||
target_vocab = self.target_model_config.hf_config.vocab_size
|
||||
draft_hf = self.draft_model_config.hf_config
|
||||
if draft_hf.vocab_size != target_vocab:
|
||||
draft_hf.vocab_size = target_vocab
|
||||
draft_hf.truncated_vocab_size = target_vocab
|
||||
|
||||
# Automatically detect the method
|
||||
if self.method in ("eagle", "eagle3", "dflash", "dspark"):
|
||||
pass
|
||||
@@ -1353,10 +1332,6 @@ class SpeculativeConfig:
|
||||
)
|
||||
):
|
||||
self.method = "dspark"
|
||||
elif self.draft_model_config.hf_config.model_type == "medusa":
|
||||
self.method = "medusa"
|
||||
elif self.draft_model_config.hf_config.model_type == "mlp_speculator":
|
||||
self.method = "mlp_speculator"
|
||||
elif self.draft_model_config.hf_config.model_type in get_args(
|
||||
MTPModelTypes
|
||||
):
|
||||
@@ -1468,13 +1443,6 @@ class SpeculativeConfig:
|
||||
if self.method in ("dflash", "dspark"):
|
||||
self.parallel_drafting = True
|
||||
|
||||
if self.num_speculative_tokens is not None and hasattr(
|
||||
self.draft_model_config.hf_config, "num_lookahead_tokens"
|
||||
):
|
||||
self.draft_model_config.hf_config.num_lookahead_tokens = (
|
||||
self.num_speculative_tokens
|
||||
)
|
||||
|
||||
n_predict = getattr(
|
||||
self.draft_model_config.hf_config, "n_predict", None
|
||||
)
|
||||
@@ -1550,7 +1518,6 @@ class SpeculativeConfig:
|
||||
SpeculativeConfig._verify_and_get_draft_tp(
|
||||
self.target_parallel_config,
|
||||
self.draft_tensor_parallel_size,
|
||||
self.draft_model_config.hf_config,
|
||||
)
|
||||
)
|
||||
self.draft_model_config.max_model_len = (
|
||||
@@ -1712,7 +1679,6 @@ class SpeculativeConfig:
|
||||
def _verify_and_get_draft_tp(
|
||||
target_parallel_config: ParallelConfig,
|
||||
speculative_draft_tensor_parallel_size: int | None,
|
||||
draft_hf_config: PreTrainedConfig,
|
||||
) -> int:
|
||||
"""Verifies and adjusts the tensor parallel size for a draft model
|
||||
specified using speculative_draft_tensor_parallel_size.
|
||||
@@ -1720,18 +1686,9 @@ class SpeculativeConfig:
|
||||
# If speculative_draft_tensor_parallel_size is unset then set it
|
||||
# appropriately else verify that it is set correctly.
|
||||
if speculative_draft_tensor_parallel_size is None:
|
||||
if draft_hf_config.model_type == "mlp_speculator":
|
||||
speculative_draft_tensor_parallel_size = 1
|
||||
if target_parallel_config.tensor_parallel_size > 1:
|
||||
logger.warning(
|
||||
"%s cannot currently be run with tp>1; "
|
||||
"setting speculative_draft_tensor_parallel_size=1",
|
||||
draft_hf_config.model_type,
|
||||
)
|
||||
else:
|
||||
speculative_draft_tensor_parallel_size = (
|
||||
target_parallel_config.tensor_parallel_size
|
||||
)
|
||||
speculative_draft_tensor_parallel_size = (
|
||||
target_parallel_config.tensor_parallel_size
|
||||
)
|
||||
elif speculative_draft_tensor_parallel_size not in (
|
||||
1,
|
||||
target_parallel_config.tensor_parallel_size,
|
||||
|
||||
+1
-6
@@ -2938,12 +2938,7 @@ class VllmConfig:
|
||||
unsupported.append("pipeline parallelism with external_launcher")
|
||||
|
||||
if speculative_config is not None:
|
||||
if speculative_config.method in (
|
||||
"suffix",
|
||||
"medusa",
|
||||
"mlp_speculator",
|
||||
"custom_class",
|
||||
):
|
||||
if speculative_config.method in ("suffix", "custom_class"):
|
||||
unsupported.append(f"speculative method '{speculative_config.method}'")
|
||||
|
||||
# V2 EagleSpeculator does not support parallel_drafting (for P-Eagle).
|
||||
|
||||
Reference in New Issue
Block a user