feat: extended EPLB support for Mistral Large 3 and additional MoE backends (#48355)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
@@ -12,23 +12,12 @@ from vllm.config import ModelConfig
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsMultiModal,
|
||||
is_mixture_of_experts,
|
||||
get_mixture_of_experts_model,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _unwrap_moe(model: nn.Module) -> nn.Module:
|
||||
# VLM wrappers (e.g. KimiK25ForConditionalGeneration) hold the MoE
|
||||
# language model under `.language_model` but don't implement
|
||||
# MixtureOfExperts themselves. Mirror the V1 path
|
||||
# (see vllm/v1/worker/gpu_model_runner.py, PR #39805).
|
||||
if not is_mixture_of_experts(model) and isinstance(model, SupportsMultiModal):
|
||||
return model.get_language_model()
|
||||
return model
|
||||
|
||||
|
||||
def step_eplb_after(*, is_dummy: bool = False) -> Callable:
|
||||
"""Step EPLB after a model runner method completes successfully."""
|
||||
|
||||
@@ -78,7 +67,8 @@ class EPLBController:
|
||||
return False
|
||||
|
||||
draft_model = speculator.model
|
||||
if not is_mixture_of_experts(draft_model):
|
||||
draft_moe_model = get_mixture_of_experts_model(draft_model)
|
||||
if draft_moe_model is None:
|
||||
return False
|
||||
|
||||
assert not self.parallel_config.enable_elastic_ep, (
|
||||
@@ -88,7 +78,7 @@ class EPLBController:
|
||||
assert speculative_config.draft_model_config is not None
|
||||
assert self.state is not None
|
||||
self.state.add_model(
|
||||
draft_model,
|
||||
draft_moe_model,
|
||||
speculative_config.draft_model_config,
|
||||
)
|
||||
speculator.set_eplb_state(self.state)
|
||||
@@ -104,13 +94,15 @@ class EPLBController:
|
||||
if not self.parallel_config.enable_eplb or load_dummy_weights:
|
||||
return False
|
||||
|
||||
model = _unwrap_moe(model)
|
||||
if not is_mixture_of_experts(model):
|
||||
moe_model = get_mixture_of_experts_model(model)
|
||||
if moe_model is None:
|
||||
return False
|
||||
|
||||
logger.info_once("EPLB is enabled for model %s.", model_config.model)
|
||||
logger.info_once(
|
||||
"EPLB is enabled for MoE part of model %s.", model_config.model
|
||||
)
|
||||
assert self.state is not None
|
||||
self.state.add_model(model, model_config)
|
||||
self.state.add_model(moe_model, model_config)
|
||||
self._has_registered_models = True
|
||||
return True
|
||||
|
||||
@@ -154,11 +146,11 @@ class EPLBController:
|
||||
expanded_physical_to_logical: torch.Tensor,
|
||||
old_num_physical_experts: int,
|
||||
) -> None:
|
||||
model = _unwrap_moe(model)
|
||||
assert is_mixture_of_experts(model)
|
||||
moe_model = get_mixture_of_experts_model(model)
|
||||
assert moe_model is not None
|
||||
|
||||
self.state = EplbState.from_mapping(
|
||||
model=model,
|
||||
model=moe_model,
|
||||
model_config=model_config,
|
||||
device=self.device,
|
||||
parallel_config=self.parallel_config,
|
||||
|
||||
Reference in New Issue
Block a user