[Mypy] Fix mypy typing for Transformers models (#54320)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-09-17 12:11:31 -04:00
committed by GitHub
co-authored by Harry Mellor
parent b78fbab4be
commit e960ead0aa
16 changed files with 224 additions and 145 deletions
+31 -11
View File
@@ -41,6 +41,7 @@ from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
from .base import Base
from .utils import log_replacement, maybe_per_layer
if TYPE_CHECKING:
@@ -66,14 +67,15 @@ class TransformersMoERunner(MoERunner):
self._moe_state = moe_state
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
def forward(
def forward( # type: ignore[override]
self,
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
**kwargs: Any,
) -> torch.Tensor:
"""In Transformers `experts.forward` will have this signature.
"""In Transformers `experts.forward` will have this signature, which is why
it deliberately does not match `MoERunner.forward`.
We discard any extra kwargs because we cannot use them here."""
# Note: we need to forward through a custom op so the topk_ids
@@ -126,10 +128,18 @@ direct_register_custom_op(
class TransformersRoutedExperts(RoutedExperts):
def get_expert_mapping(
self, include_fused: bool = False
self,
ckpt_gate_proj_name: str | None = None,
ckpt_down_proj_name: str | None = None,
ckpt_up_proj_name: str | None = None,
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
common_names = ("gate_proj", "down_proj", "up_proj")
common_map = super().get_expert_mapping(*common_names, include_fused)
common_map = super().get_expert_mapping(
ckpt_gate_proj_name or "gate_proj",
ckpt_down_proj_name or "down_proj",
ckpt_up_proj_name or "up_proj",
include_fused,
)
mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused)
if not include_fused:
return common_map + mixtral_map
@@ -138,7 +148,13 @@ class TransformersRoutedExperts(RoutedExperts):
return common_fused + mixtral_fused + common_unfused + mixtral_unfused
class MoEMixin(MixtureOfExperts):
class MoEMixin(MixtureOfExperts, Base):
mlp_layers: list[nn.Module]
"""MoE blocks whose experts were replaced, for the `MixtureOfExperts` methods."""
# Redeclared because `recursive_replace` assigns it below the first read
num_local_physical_experts: int
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
self.check_version("5.0.0", "MoE models support")
# Skip MixtureOfExperts.__init__ and call the next class in MRO
@@ -197,7 +213,8 @@ class MoEMixin(MixtureOfExperts):
# Dtype the router computes in, if it is not the activation dtype.
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
if config_router_dtype is not None:
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
# Grouped topk routing kwargs
num_expert_group = getattr(text_config, "n_group", None)
@@ -222,7 +239,7 @@ class MoEMixin(MixtureOfExperts):
# MixtureOfExperts mixin settings
ep_size = get_ep_group().world_size
self.mlp_layers = [] # Used for MixtureOfExperts methods
self.mlp_layers = []
self.moe_layers = []
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
self.num_logical_experts = num_experts
@@ -321,9 +338,10 @@ class MoEMixin(MixtureOfExperts):
if shared_experts is not None:
hf_shared = shared_experts.shared_experts
glu_fuser = get_fuser(hf_shared, GLUFuser)
down_name = glu_fuser and glu_fuser.down_name
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
if glu_fuser is not None:
down_name = glu_fuser.down_name
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
# Prefer config, otherwise read it from fuser.
router_dtype = config_router_dtype or fuser.router_dtype
gate = fuser.gate(moe_block, prefix, router_dtype)
@@ -374,7 +392,9 @@ class MoEMixin(MixtureOfExperts):
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
is_sp = moe_state.is_sequence_parallel
group = get_ep_group() if is_sp else get_dp_group()
assert sizes[group.rank_in_group] == topk_ids.shape[0]