[Feature] Support custom callable proposer backend for speculative decoding (#39487)
Signed-off-by: 524031910363 <hyzhyzsh@sjtu.edu.cn> Signed-off-by: CynicDora <hyzhyzsh@sjtu.edu.cn>
This commit is contained in:
@@ -62,6 +62,7 @@ SpeculativeMethod = Literal[
|
||||
"mlp_speculator",
|
||||
"draft_model",
|
||||
"suffix",
|
||||
"custom_class",
|
||||
EagleModelTypes,
|
||||
NgramGPUTypes,
|
||||
]
|
||||
@@ -511,7 +512,16 @@ class SpeculativeConfig:
|
||||
# default.
|
||||
|
||||
# infer method from user args
|
||||
if self.method is None:
|
||||
# Check if the model field contains a custom module path (e.g., 'pkg.Mod')
|
||||
if (
|
||||
self.model is not None
|
||||
and "." in self.model
|
||||
and not self.model.startswith(("http://", "https://", "file://"))
|
||||
and "/" not in self.model # not a HuggingFace repo (org/model)
|
||||
):
|
||||
# Treat as a custom class path
|
||||
self.method = "custom_class"
|
||||
elif self.method is None:
|
||||
if self.model in ("ngram", "[ngram]"):
|
||||
self.method = "ngram"
|
||||
else:
|
||||
@@ -545,6 +555,14 @@ class SpeculativeConfig:
|
||||
self.model = "suffix"
|
||||
elif self.method == "extract_hidden_states":
|
||||
self.model = "extract_hidden_states"
|
||||
elif self.method == "custom_class":
|
||||
# method was set explicitly, but model should already contain the
|
||||
# custom module path. If not, this is a configuration error.
|
||||
if self.model is None:
|
||||
raise ValueError(
|
||||
"method='custom_class' requires 'model' to contain the "
|
||||
"custom proposer module path (e.g., 'my_module.MyProposer')."
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"num_speculative_tokens was provided but without speculative model."
|
||||
@@ -588,6 +606,18 @@ class SpeculativeConfig:
|
||||
self.draft_parallel_config = self.target_parallel_config
|
||||
elif self.method == "suffix":
|
||||
self._validate_suffix_decoding()
|
||||
elif self.method == "custom_class":
|
||||
# Custom class proposer does not need a draft model.
|
||||
# It will dynamically load the user-provided class at runtime.
|
||||
logger.warning_once(
|
||||
"Using a custom class-based proposer backend. This is an "
|
||||
"experimental feature and the proposer interface is subject to "
|
||||
"breaking changes in future vLLM releases."
|
||||
)
|
||||
self.prompt_lookup_max = 0
|
||||
self.prompt_lookup_min = 0
|
||||
self.draft_model_config = self.target_model_config
|
||||
self.draft_parallel_config = self.target_parallel_config
|
||||
elif self.method == "extract_hidden_states":
|
||||
from vllm.transformers_utils.configs.extract_hidden_states import (
|
||||
ExtractHiddenStatesConfig,
|
||||
@@ -1063,7 +1093,13 @@ class SpeculativeConfig:
|
||||
method = self.method
|
||||
model = (
|
||||
None
|
||||
if method in ("ngram", "suffix", "extract_hidden_states")
|
||||
if method
|
||||
in (
|
||||
"ngram",
|
||||
"suffix",
|
||||
"extract_hidden_states",
|
||||
"custom_class",
|
||||
)
|
||||
else self.draft_model_config.model
|
||||
)
|
||||
num_spec_tokens = self.num_speculative_tokens
|
||||
|
||||
@@ -1613,11 +1613,6 @@ class EngineArgs:
|
||||
) -> SpeculativeConfig | None:
|
||||
"""Initializes and returns a SpeculativeConfig object based on
|
||||
`speculative_config`.
|
||||
|
||||
This function utilizes `speculative_config` to create a
|
||||
SpeculativeConfig object. The `speculative_config` can either be
|
||||
provided as a JSON string input via CLI arguments or directly as a
|
||||
dictionary from the engine.
|
||||
"""
|
||||
if self.speculative_config is None:
|
||||
return None
|
||||
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def create_custom_proposer(vllm_config: VllmConfig):
|
||||
"""Load and instantiate a user-provided proposer class.
|
||||
|
||||
The class path is read from ``speculative_config.model``
|
||||
(e.g., ``"my_module.MyCustomProposer"``). The class is
|
||||
imported, instantiated with *vllm_config*, and returned
|
||||
directly so the caller can use it without any wrapper.
|
||||
|
||||
The returned object must expose a callable ``propose`` method.
|
||||
"""
|
||||
assert vllm_config.speculative_config is not None
|
||||
spec_config = vllm_config.speculative_config
|
||||
|
||||
backend = spec_config.model
|
||||
assert backend is not None
|
||||
|
||||
if "." not in backend:
|
||||
raise ValueError(
|
||||
f"Invalid custom proposer module path '{backend}'. "
|
||||
"It must be a full module path (e.g., 'module.MyProposerClass')."
|
||||
)
|
||||
|
||||
module_path, class_name = backend.rsplit(".", 1)
|
||||
try:
|
||||
module = importlib.import_module(module_path)
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
f"Cannot import module '{module_path}' for custom proposer '{backend}': {e}"
|
||||
) from e
|
||||
|
||||
user_class = getattr(module, class_name, None)
|
||||
if user_class is None:
|
||||
raise AttributeError(
|
||||
f"Module '{module_path}' has no attribute '{class_name}' "
|
||||
f"(speculative_config.model='{backend}')"
|
||||
)
|
||||
|
||||
try:
|
||||
instance = user_class(vllm_config)
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"Failed to instantiate custom proposer class '{backend}': {e}. "
|
||||
"The class constructor must accept VllmConfig as argument."
|
||||
) from e
|
||||
|
||||
if not hasattr(instance, "propose"):
|
||||
raise AttributeError(
|
||||
f"Custom proposer class '{backend}' must have a 'propose' method."
|
||||
)
|
||||
if not callable(instance.propose):
|
||||
raise AttributeError(
|
||||
f"Custom proposer class '{backend}' has a 'propose' attribute "
|
||||
"but it is not callable."
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Loaded custom proposer class '%s' with num_speculative_tokens=%d",
|
||||
backend,
|
||||
spec_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
return instance
|
||||
@@ -169,6 +169,7 @@ from vllm.v1.sample.logits_processor.interface import LogitsProcessor
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.rejection_sampler import RejectionSampler
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
from vllm.v1.spec_decode.custom_class_proposer import create_custom_proposer
|
||||
from vllm.v1.spec_decode.dflash import DFlashProposer
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
@@ -531,7 +532,11 @@ class GPUModelRunner(
|
||||
| ExtractHiddenStatesProposer
|
||||
| Gemma4Proposer
|
||||
)
|
||||
if self.speculative_config.method == "ngram":
|
||||
if self.speculative_config.method == "custom_class":
|
||||
self.drafter = create_custom_proposer( # type: ignore[assignment]
|
||||
self.vllm_config
|
||||
)
|
||||
elif self.speculative_config.method == "ngram":
|
||||
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
|
||||
|
||||
self.drafter = NgramProposer(self.vllm_config)
|
||||
@@ -4614,6 +4619,14 @@ class GPUModelRunner(
|
||||
self.input_batch.token_ids_cpu,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.method == "custom_class":
|
||||
assert isinstance(sampled_token_ids, list)
|
||||
draft_token_ids = cast(Any, self.drafter).propose(
|
||||
sampled_token_ids,
|
||||
self.input_batch.num_tokens_no_spec,
|
||||
self.input_batch.token_ids_cpu,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.use_ngram_gpu():
|
||||
assert isinstance(self.drafter, NgramProposerGPU)
|
||||
(
|
||||
@@ -4885,7 +4898,8 @@ class GPUModelRunner(
|
||||
)
|
||||
if hasattr(self, "drafter"):
|
||||
logger.info_once("Loading drafter model...")
|
||||
self.drafter.load_model(self.model)
|
||||
if hasattr(self.drafter, "load_model"):
|
||||
self.drafter.load_model(self.model)
|
||||
if (
|
||||
hasattr(self.drafter, "model")
|
||||
and is_mixture_of_experts(self.drafter.model)
|
||||
|
||||
Reference in New Issue
Block a user