[Feature] Support per-draft-model MoE backend via --speculative-config (#37880)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: [Andrii Skliar] <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-03-25 14:31:52 +00:00
committed by GitHub
co-authored by Andrii Skliar
parent a1a2566447
commit cd7643015e
5 changed files with 140 additions and 40 deletions
+18 -5
View File
@@ -6,10 +6,10 @@ import torch.nn as nn
from typing_extensions import override
from vllm.config import VllmConfig
from vllm.config.utils import replace
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.v1.spec_decode.eagle import SpecDecodeBaseProposer
from vllm.v1.spec_decode.utils import create_vllm_config_for_draft_model
logger = init_logger(__name__)
@@ -50,16 +50,29 @@ class DraftModelProposer(SpecDecodeBaseProposer):
"Please pass 'draft_tensor_parallel_size' in the speculative_config."
)
@override
def _create_draft_vllm_config(self) -> VllmConfig:
base = super()._create_draft_vllm_config()
spec = self.speculative_config
return replace(
base,
quant_config=None,
parallel_config=replace(
spec.draft_parallel_config,
rank=self.vllm_config.parallel_config.rank,
),
model_config=spec.draft_model_config,
)
@override
def _get_model(self) -> nn.Module:
# Draft models may be quantized or on different parallelism,
# so we load them with a modified vllm config
from vllm.compilation.backends import set_model_tag
temp_vllm_config = create_vllm_config_for_draft_model(self.vllm_config)
draft_vllm_config = self._create_draft_vllm_config()
with set_model_tag("draft_model"):
model = get_model(
vllm_config=temp_vllm_config,
vllm_config=draft_vllm_config,
prefix="draft_model",
)
return model