feat: extended EPLB support for Mistral Large 3 and additional MoE backends (#48355)

Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
Julien Debache
2026-08-07 05:32:52 -07:00
committed by GitHub
parent 8d9b52f7c2
commit ae934ba8a5
16 changed files with 500 additions and 96 deletions
+13 -21
View File
@@ -12,23 +12,12 @@ from vllm.config import ModelConfig
from vllm.distributed.eplb.eplb_state import EplbState
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
SupportsMultiModal,
is_mixture_of_experts,
get_mixture_of_experts_model,
)
logger = init_logger(__name__)
def _unwrap_moe(model: nn.Module) -> nn.Module:
# VLM wrappers (e.g. KimiK25ForConditionalGeneration) hold the MoE
# language model under `.language_model` but don't implement
# MixtureOfExperts themselves. Mirror the V1 path
# (see vllm/v1/worker/gpu_model_runner.py, PR #39805).
if not is_mixture_of_experts(model) and isinstance(model, SupportsMultiModal):
return model.get_language_model()
return model
def step_eplb_after(*, is_dummy: bool = False) -> Callable:
"""Step EPLB after a model runner method completes successfully."""
@@ -78,7 +67,8 @@ class EPLBController:
return False
draft_model = speculator.model
if not is_mixture_of_experts(draft_model):
draft_moe_model = get_mixture_of_experts_model(draft_model)
if draft_moe_model is None:
return False
assert not self.parallel_config.enable_elastic_ep, (
@@ -88,7 +78,7 @@ class EPLBController:
assert speculative_config.draft_model_config is not None
assert self.state is not None
self.state.add_model(
draft_model,
draft_moe_model,
speculative_config.draft_model_config,
)
speculator.set_eplb_state(self.state)
@@ -104,13 +94,15 @@ class EPLBController:
if not self.parallel_config.enable_eplb or load_dummy_weights:
return False
model = _unwrap_moe(model)
if not is_mixture_of_experts(model):
moe_model = get_mixture_of_experts_model(model)
if moe_model is None:
return False
logger.info_once("EPLB is enabled for model %s.", model_config.model)
logger.info_once(
"EPLB is enabled for MoE part of model %s.", model_config.model
)
assert self.state is not None
self.state.add_model(model, model_config)
self.state.add_model(moe_model, model_config)
self._has_registered_models = True
return True
@@ -154,11 +146,11 @@ class EPLBController:
expanded_physical_to_logical: torch.Tensor,
old_num_physical_experts: int,
) -> None:
model = _unwrap_moe(model)
assert is_mixture_of_experts(model)
moe_model = get_mixture_of_experts_model(model)
assert moe_model is not None
self.state = EplbState.from_mapping(
model=model,
model=moe_model,
model_config=model_config,
device=self.device,
parallel_config=self.parallel_config,