[Feature] Support custom callable proposer backend for speculative decoding (#39487)

Signed-off-by: 524031910363 <hyzhyzsh@sjtu.edu.cn>
Signed-off-by: CynicDora <hyzhyzsh@sjtu.edu.cn>
This commit is contained in:
CynicDora
2026-05-13 16:53:01 +00:00
committed by GitHub
parent e35c0d4c63
commit 256dbcaabf
7 changed files with 261 additions and 11 deletions
+16 -2
View File
@@ -169,6 +169,7 @@ from vllm.v1.sample.logits_processor.interface import LogitsProcessor
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.rejection_sampler import RejectionSampler
from vllm.v1.sample.sampler import Sampler
from vllm.v1.spec_decode.custom_class_proposer import create_custom_proposer
from vllm.v1.spec_decode.dflash import DFlashProposer
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
@@ -531,7 +532,11 @@ class GPUModelRunner(
| ExtractHiddenStatesProposer
| Gemma4Proposer
)
if self.speculative_config.method == "ngram":
if self.speculative_config.method == "custom_class":
self.drafter = create_custom_proposer( # type: ignore[assignment]
self.vllm_config
)
elif self.speculative_config.method == "ngram":
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
self.drafter = NgramProposer(self.vllm_config)
@@ -4614,6 +4619,14 @@ class GPUModelRunner(
self.input_batch.token_ids_cpu,
slot_mappings=slot_mappings,
)
elif spec_config.method == "custom_class":
assert isinstance(sampled_token_ids, list)
draft_token_ids = cast(Any, self.drafter).propose(
sampled_token_ids,
self.input_batch.num_tokens_no_spec,
self.input_batch.token_ids_cpu,
slot_mappings=slot_mappings,
)
elif spec_config.use_ngram_gpu():
assert isinstance(self.drafter, NgramProposerGPU)
(
@@ -4885,7 +4898,8 @@ class GPUModelRunner(
)
if hasattr(self, "drafter"):
logger.info_once("Loading drafter model...")
self.drafter.load_model(self.model)
if hasattr(self.drafter, "load_model"):
self.drafter.load_model(self.model)
if (
hasattr(self.drafter, "model")
and is_mixture_of_experts(self.drafter.model)