[Mypy] Fix mypy typing for Transformers models (#54320)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
b78fbab4be
commit
e960ead0aa
@@ -41,6 +41,7 @@ from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
|
||||
|
||||
from .base import Base
|
||||
from .utils import log_replacement, maybe_per_layer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -66,14 +67,15 @@ class TransformersMoERunner(MoERunner):
|
||||
self._moe_state = moe_state
|
||||
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor:
|
||||
"""In Transformers `experts.forward` will have this signature.
|
||||
"""In Transformers `experts.forward` will have this signature, which is why
|
||||
it deliberately does not match `MoERunner.forward`.
|
||||
|
||||
We discard any extra kwargs because we cannot use them here."""
|
||||
# Note: we need to forward through a custom op so the topk_ids
|
||||
@@ -126,10 +128,18 @@ direct_register_custom_op(
|
||||
|
||||
class TransformersRoutedExperts(RoutedExperts):
|
||||
def get_expert_mapping(
|
||||
self, include_fused: bool = False
|
||||
self,
|
||||
ckpt_gate_proj_name: str | None = None,
|
||||
ckpt_down_proj_name: str | None = None,
|
||||
ckpt_up_proj_name: str | None = None,
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
common_names = ("gate_proj", "down_proj", "up_proj")
|
||||
common_map = super().get_expert_mapping(*common_names, include_fused)
|
||||
common_map = super().get_expert_mapping(
|
||||
ckpt_gate_proj_name or "gate_proj",
|
||||
ckpt_down_proj_name or "down_proj",
|
||||
ckpt_up_proj_name or "up_proj",
|
||||
include_fused,
|
||||
)
|
||||
mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused)
|
||||
if not include_fused:
|
||||
return common_map + mixtral_map
|
||||
@@ -138,7 +148,13 @@ class TransformersRoutedExperts(RoutedExperts):
|
||||
return common_fused + mixtral_fused + common_unfused + mixtral_unfused
|
||||
|
||||
|
||||
class MoEMixin(MixtureOfExperts):
|
||||
class MoEMixin(MixtureOfExperts, Base):
|
||||
mlp_layers: list[nn.Module]
|
||||
"""MoE blocks whose experts were replaced, for the `MixtureOfExperts` methods."""
|
||||
|
||||
# Redeclared because `recursive_replace` assigns it below the first read
|
||||
num_local_physical_experts: int
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
self.check_version("5.0.0", "MoE models support")
|
||||
# Skip MixtureOfExperts.__init__ and call the next class in MRO
|
||||
@@ -197,7 +213,8 @@ class MoEMixin(MixtureOfExperts):
|
||||
|
||||
# Dtype the router computes in, if it is not the activation dtype.
|
||||
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
|
||||
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
|
||||
if config_router_dtype is not None:
|
||||
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
|
||||
|
||||
# Grouped topk routing kwargs
|
||||
num_expert_group = getattr(text_config, "n_group", None)
|
||||
@@ -222,7 +239,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
# MixtureOfExperts mixin settings
|
||||
ep_size = get_ep_group().world_size
|
||||
|
||||
self.mlp_layers = [] # Used for MixtureOfExperts methods
|
||||
self.mlp_layers = []
|
||||
self.moe_layers = []
|
||||
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
|
||||
self.num_logical_experts = num_experts
|
||||
@@ -321,9 +338,10 @@ class MoEMixin(MixtureOfExperts):
|
||||
if shared_experts is not None:
|
||||
hf_shared = shared_experts.shared_experts
|
||||
glu_fuser = get_fuser(hf_shared, GLUFuser)
|
||||
down_name = glu_fuser and glu_fuser.down_name
|
||||
if down_name is not None:
|
||||
shared_down_projs.append((hf_shared, down_name))
|
||||
if glu_fuser is not None:
|
||||
down_name = glu_fuser.down_name
|
||||
if down_name is not None:
|
||||
shared_down_projs.append((hf_shared, down_name))
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
router_dtype = config_router_dtype or fuser.router_dtype
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
@@ -374,7 +392,9 @@ class MoEMixin(MixtureOfExperts):
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[group.rank_in_group] == topk_ids.shape[0]
|
||||
|
||||
Reference in New Issue
Block a user