feat: extended EPLB support for Mistral Large 3 and additional MoE backends (#48355)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
@@ -82,7 +82,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsXDRoPE,
|
||||
is_mixture_of_experts,
|
||||
get_mixture_of_experts_model,
|
||||
supports_eagle3,
|
||||
supports_mrope,
|
||||
supports_multimodal_pruning,
|
||||
@@ -5364,9 +5364,14 @@ class GPUModelRunner(
|
||||
if hasattr(self.drafter, "load_model"):
|
||||
self.drafter.load_model(self.model)
|
||||
if (
|
||||
hasattr(self.drafter, "model")
|
||||
and is_mixture_of_experts(self.drafter.model)
|
||||
and self.parallel_config.enable_eplb
|
||||
self.parallel_config.enable_eplb
|
||||
and hasattr(self.drafter, "model")
|
||||
and (
|
||||
drafter_moe_model := get_mixture_of_experts_model(
|
||||
self.drafter.model
|
||||
)
|
||||
)
|
||||
is not None
|
||||
):
|
||||
assert not self.parallel_config.enable_elastic_ep, (
|
||||
"Elastic EP is not supported with drafter model."
|
||||
@@ -5375,7 +5380,7 @@ class GPUModelRunner(
|
||||
assert spec_config is not None
|
||||
assert spec_config.draft_model_config is not None
|
||||
logger.info_once(
|
||||
"EPLB is enabled for drafter model %s.",
|
||||
"EPLB is enabled for MoE part of drafter model %s.",
|
||||
spec_config.draft_model_config.model,
|
||||
)
|
||||
if self.eplb_state is None:
|
||||
@@ -5383,7 +5388,7 @@ class GPUModelRunner(
|
||||
self.parallel_config, self.device
|
||||
)
|
||||
self.eplb_state.add_model(
|
||||
self.drafter.model,
|
||||
drafter_moe_model,
|
||||
spec_config.draft_model_config,
|
||||
)
|
||||
assert hasattr(self.drafter, "set_eplb_state")
|
||||
@@ -5396,21 +5401,15 @@ class GPUModelRunner(
|
||||
# VLM models (e.g. KimiK25ForConditionalGeneration) wrap the
|
||||
# actual MoE language model but don't implement
|
||||
# MixtureOfExperts themselves.
|
||||
moe_candidate = self.model
|
||||
if not is_mixture_of_experts(moe_candidate) and isinstance(
|
||||
moe_candidate, SupportsMultiModal
|
||||
):
|
||||
moe_candidate = moe_candidate.get_language_model()
|
||||
if is_mixture_of_experts(moe_candidate):
|
||||
self._moe_model = moe_candidate
|
||||
self._moe_model = get_mixture_of_experts_model(self.model)
|
||||
|
||||
if (
|
||||
self._moe_model is not None
|
||||
and self.parallel_config.enable_eplb
|
||||
self.parallel_config.enable_eplb
|
||||
and not load_dummy_weights
|
||||
and self._moe_model is not None
|
||||
):
|
||||
logger.info_once(
|
||||
"EPLB is enabled for model %s.",
|
||||
"EPLB is enabled for MoE part of model %s.",
|
||||
self.model_config.model,
|
||||
)
|
||||
assert self.eplb_state is not None
|
||||
@@ -5450,8 +5449,8 @@ class GPUModelRunner(
|
||||
) # Temporary hack for dynamic res video w/o support for bs>1 yet
|
||||
|
||||
if (
|
||||
self._moe_model is not None
|
||||
and self.parallel_config.enable_eplb
|
||||
self.parallel_config.enable_eplb
|
||||
and self._moe_model is not None
|
||||
and not load_dummy_weights
|
||||
and self.eplb_state is not None
|
||||
and self.eplb_state.is_async
|
||||
|
||||
Reference in New Issue
Block a user