feat: extended EPLB support for Mistral Large 3 and additional MoE backends (#48355)

Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
Julien Debache
2026-08-07 05:32:52 -07:00
committed by GitHub
parent 8d9b52f7c2
commit ae934ba8a5
16 changed files with 500 additions and 96 deletions
+32 -2
View File
@@ -5,7 +5,6 @@ import asyncio
from collections.abc import (
AsyncGenerator,
Callable,
Iterable,
Mapping,
MutableSequence,
Sequence,
@@ -949,7 +948,7 @@ class MixtureOfExperts(Protocol):
num_redundant_experts: int
"""Number of redundant experts in this model."""
moe_layers: Iterable["MoERunner"]
moe_layers: Sequence["MoERunner"]
"""List of MoE layers in this model."""
def set_eplb_state(
@@ -992,6 +991,37 @@ class MixtureOfExperts(Protocol):
) -> None: ...
def get_mixture_of_experts_model(model: object) -> MixtureOfExperts | None:
"""Return the MixtureOfExperts contained within an arbitrary model.
- If the model itself is a MixtureOfExperts, return the model directly.
- If the model is a multi-modal model, and its `language_model` is a
MixtureOfExperts, return the `language_model`.
- If neither, return None.
Args:
model: Model being served.
Returns:
The MixtureOfExperts instance contained within the model, or None.
"""
if is_mixture_of_experts(model):
return model
if isinstance(model, SupportsMultiModal):
try:
mm_language_model = model.get_language_model()
return (
mm_language_model if is_mixture_of_experts(mm_language_model) else None
)
except NotImplementedError:
logger.info_once("Cannot fetch language_model from MultiModal model")
return None
return None
def is_mixture_of_experts(model: object) -> TypeIs[MixtureOfExperts]:
return (
isinstance(model, MixtureOfExperts) and getattr(model, "num_moe_layers", 0) > 0