[Feature] Support custom callable proposer backend for speculative decoding (#39487)

Signed-off-by: 524031910363 <hyzhyzsh@sjtu.edu.cn>
Signed-off-by: CynicDora <hyzhyzsh@sjtu.edu.cn>
This commit is contained in:
CynicDora
2026-05-13 16:53:01 +00:00
committed by GitHub
parent e35c0d4c63
commit 256dbcaabf
7 changed files with 261 additions and 11 deletions
+38 -2
View File
@@ -62,6 +62,7 @@ SpeculativeMethod = Literal[
"mlp_speculator",
"draft_model",
"suffix",
"custom_class",
EagleModelTypes,
NgramGPUTypes,
]
@@ -511,7 +512,16 @@ class SpeculativeConfig:
# default.
# infer method from user args
if self.method is None:
# Check if the model field contains a custom module path (e.g., 'pkg.Mod')
if (
self.model is not None
and "." in self.model
and not self.model.startswith(("http://", "https://", "file://"))
and "/" not in self.model # not a HuggingFace repo (org/model)
):
# Treat as a custom class path
self.method = "custom_class"
elif self.method is None:
if self.model in ("ngram", "[ngram]"):
self.method = "ngram"
else:
@@ -545,6 +555,14 @@ class SpeculativeConfig:
self.model = "suffix"
elif self.method == "extract_hidden_states":
self.model = "extract_hidden_states"
elif self.method == "custom_class":
# method was set explicitly, but model should already contain the
# custom module path. If not, this is a configuration error.
if self.model is None:
raise ValueError(
"method='custom_class' requires 'model' to contain the "
"custom proposer module path (e.g., 'my_module.MyProposer')."
)
else:
raise ValueError(
"num_speculative_tokens was provided but without speculative model."
@@ -588,6 +606,18 @@ class SpeculativeConfig:
self.draft_parallel_config = self.target_parallel_config
elif self.method == "suffix":
self._validate_suffix_decoding()
elif self.method == "custom_class":
# Custom class proposer does not need a draft model.
# It will dynamically load the user-provided class at runtime.
logger.warning_once(
"Using a custom class-based proposer backend. This is an "
"experimental feature and the proposer interface is subject to "
"breaking changes in future vLLM releases."
)
self.prompt_lookup_max = 0
self.prompt_lookup_min = 0
self.draft_model_config = self.target_model_config
self.draft_parallel_config = self.target_parallel_config
elif self.method == "extract_hidden_states":
from vllm.transformers_utils.configs.extract_hidden_states import (
ExtractHiddenStatesConfig,
@@ -1063,7 +1093,13 @@ class SpeculativeConfig:
method = self.method
model = (
None
if method in ("ngram", "suffix", "extract_hidden_states")
if method
in (
"ngram",
"suffix",
"extract_hidden_states",
"custom_class",
)
else self.draft_model_config.model
)
num_spec_tokens = self.num_speculative_tokens
-5
View File
@@ -1613,11 +1613,6 @@ class EngineArgs:
) -> SpeculativeConfig | None:
"""Initializes and returns a SpeculativeConfig object based on
`speculative_config`.
This function utilizes `speculative_config` to create a
SpeculativeConfig object. The `speculative_config` can either be
provided as a JSON string input via CLI arguments or directly as a
dictionary from the engine.
"""
if self.speculative_config is None:
return None
+73
View File
@@ -0,0 +1,73 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
from vllm.config import VllmConfig
from vllm.logger import init_logger
logger = init_logger(__name__)
def create_custom_proposer(vllm_config: VllmConfig):
"""Load and instantiate a user-provided proposer class.
The class path is read from ``speculative_config.model``
(e.g., ``"my_module.MyCustomProposer"``). The class is
imported, instantiated with *vllm_config*, and returned
directly so the caller can use it without any wrapper.
The returned object must expose a callable ``propose`` method.
"""
assert vllm_config.speculative_config is not None
spec_config = vllm_config.speculative_config
backend = spec_config.model
assert backend is not None
if "." not in backend:
raise ValueError(
f"Invalid custom proposer module path '{backend}'. "
"It must be a full module path (e.g., 'module.MyProposerClass')."
)
module_path, class_name = backend.rsplit(".", 1)
try:
module = importlib.import_module(module_path)
except ImportError as e:
raise ImportError(
f"Cannot import module '{module_path}' for custom proposer '{backend}': {e}"
) from e
user_class = getattr(module, class_name, None)
if user_class is None:
raise AttributeError(
f"Module '{module_path}' has no attribute '{class_name}' "
f"(speculative_config.model='{backend}')"
)
try:
instance = user_class(vllm_config)
except Exception as e:
raise RuntimeError(
f"Failed to instantiate custom proposer class '{backend}': {e}. "
"The class constructor must accept VllmConfig as argument."
) from e
if not hasattr(instance, "propose"):
raise AttributeError(
f"Custom proposer class '{backend}' must have a 'propose' method."
)
if not callable(instance.propose):
raise AttributeError(
f"Custom proposer class '{backend}' has a 'propose' attribute "
"but it is not callable."
)
logger.info(
"Loaded custom proposer class '%s' with num_speculative_tokens=%d",
backend,
spec_config.num_speculative_tokens,
)
return instance
+16 -2
View File
@@ -169,6 +169,7 @@ from vllm.v1.sample.logits_processor.interface import LogitsProcessor
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.rejection_sampler import RejectionSampler
from vllm.v1.sample.sampler import Sampler
from vllm.v1.spec_decode.custom_class_proposer import create_custom_proposer
from vllm.v1.spec_decode.dflash import DFlashProposer
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
@@ -531,7 +532,11 @@ class GPUModelRunner(
| ExtractHiddenStatesProposer
| Gemma4Proposer
)
if self.speculative_config.method == "ngram":
if self.speculative_config.method == "custom_class":
self.drafter = create_custom_proposer( # type: ignore[assignment]
self.vllm_config
)
elif self.speculative_config.method == "ngram":
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
self.drafter = NgramProposer(self.vllm_config)
@@ -4614,6 +4619,14 @@ class GPUModelRunner(
self.input_batch.token_ids_cpu,
slot_mappings=slot_mappings,
)
elif spec_config.method == "custom_class":
assert isinstance(sampled_token_ids, list)
draft_token_ids = cast(Any, self.drafter).propose(
sampled_token_ids,
self.input_batch.num_tokens_no_spec,
self.input_batch.token_ids_cpu,
slot_mappings=slot_mappings,
)
elif spec_config.use_ngram_gpu():
assert isinstance(self.drafter, NgramProposerGPU)
(
@@ -4885,7 +4898,8 @@ class GPUModelRunner(
)
if hasattr(self, "drafter"):
logger.info_once("Loading drafter model...")
self.drafter.load_model(self.model)
if hasattr(self.drafter, "load_model"):
self.drafter.load_model(self.model)
if (
hasattr(self.drafter, "model")
and is_mixture_of_experts(self.drafter.model)