[Mypy] Fix typing for R/S models (#54157)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-10 09:43:08 -04:00
committed by GitHub
parent 73fb19151f
commit f9083cb83a
15 changed files with 146 additions and 85 deletions
+39 -29
View File
@@ -11,7 +11,7 @@ from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_dp_group,
get_ep_group,
@@ -47,6 +47,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
if typing.TYPE_CHECKING:
from vllm.transformers_utils.configs.step3p5 import Step3p5Config
else:
from transformers import PretrainedConfig as Step3p5Config
from .interfaces import MixtureOfExperts, SupportsPP
from .utils import (
AutoWeightsLoader,
@@ -79,7 +84,7 @@ class FP32ReplicatedLinear(ReplicatedLinear):
class Step3p5MLP(nn.Module):
def __init__(
self,
config: ModelConfig,
config: Step3p5Config,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
@@ -147,9 +152,9 @@ class Step3p5Attention(nn.Module):
# Step3p5 specific args
sliding_window: int | None = None,
use_head_wise_attn_gate: bool = False,
layer_types: list = None,
use_rope_layers: list = None,
yarn_only_types: list = None,
layer_types: list[str] | None = None,
use_rope_layers: list[bool] | None = None,
yarn_only_types: list[str] | None = None,
swa_num_attention_heads: int | None = None,
partial_rotary_factor: float = 1.0,
):
@@ -162,8 +167,10 @@ class Step3p5Attention(nn.Module):
enable_sliding_window = layer_types[self.layer_idx] == "sliding_attention"
else:
enable_sliding_window = self.layer_idx % 2 == 0
if yarn_only_types and layer_types[self.layer_idx] not in yarn_only_types:
rope_scaling = None
if yarn_only_types:
assert layer_types is not None
if layer_types[self.layer_idx] not in yarn_only_types:
rope_scaling = None
if sliding_window is not None and enable_sliding_window:
sliding_window = sliding_window
@@ -304,6 +311,7 @@ class FusedMoEBlock(nn.Module):
self.ep_size = get_ep_group().device_group.size()
config = vllm_config.model_config.hf_config
assert isinstance(config, Step3p5Config)
quant_config = vllm_config.quant_config
parallel_config = vllm_config.parallel_config
@@ -349,6 +357,7 @@ class FusedMoEBlock(nn.Module):
else None
)
if swiglu_limit not in (None, 0):
assert swiglu_limit is not None
swiglu_limit = float(swiglu_limit)
assert swiglu_limit == 7.0, (
"Swiglu limit in fused moe block only support 7.0 now."
@@ -408,6 +417,7 @@ class Step3p5DecoderLayer(nn.Module):
) -> None:
super().__init__()
config = vllm_config.model_config.hf_config
assert isinstance(config, Step3p5Config)
self.hidden_size = config.hidden_size
layer_idx = extract_layer_index(prefix)
self.layer_idx = layer_idx
@@ -419,20 +429,18 @@ class Step3p5DecoderLayer(nn.Module):
num_attention_heads = None
num_attention_groups = None
head_dim = None
attention_other_setting = config.attention_other_setting
layer_types = config.layer_types
if (
getattr(config, "attention_other_setting", None)
and getattr(config, "layer_types", [])
and config.layer_types[layer_idx]
== config.attention_other_setting["attention_type"]
attention_other_setting
and layer_types
and layer_types[layer_idx] == attention_other_setting["attention_type"]
):
num_attention_heads = config.attention_other_setting[
"num_attention_heads"
]
num_attention_groups = config.attention_other_setting[
"num_attention_groups"
]
head_dim = config.attention_other_setting["head_dim"]
num_attention_heads = attention_other_setting["num_attention_heads"]
num_attention_groups = attention_other_setting["num_attention_groups"]
head_dim = attention_other_setting["head_dim"]
partial_rotary_factors = getattr(config, "partial_rotary_factors", [])
assert config.max_position_embeddings is not None
self.self_attn = Step3p5Attention(
hidden_size=self.hidden_size,
num_heads=num_attention_heads
@@ -539,6 +547,7 @@ class Step3p5Model(nn.Module):
self.vllm_config = vllm_config
config = vllm_config.model_config.hf_config
assert isinstance(config, Step3p5Config)
self.vocab_size = config.vocab_size
self.config = config
@@ -605,8 +614,8 @@ class Step3p5Model(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
config = self.config
assert config.num_attention_groups > 1, "Only support GQA"
qkv_params_mapping = []
stacked_params_mapping = [
qkv_params_mapping: list[tuple[str, str, float, float]] = []
stacked_params_mapping: list[tuple[str, str, str | int]] = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
@@ -622,7 +631,7 @@ class Step3p5Model(nn.Module):
)
# Old packed 3D format: .moe.gate_proj.weight [num_experts, out, in]
expert_params_mapping = [
expert_params_mapping: list[tuple[str, str, str]] = [
(
f".moe.experts.routed_experts.{base_layer}w13_weight",
".moe.gate_proj.weight",
@@ -758,16 +767,16 @@ class Step3p5Model(nn.Module):
# Not an expert proj — use default loader
# (e.g. share_expert weights if they matched)
param = params_dict[local_name]
weight_loader = getattr(
fallback_loader = getattr(
param,
"weight_loader",
default_weight_loader,
)
weight_loader(param, loaded_weight)
fallback_loader(param, loaded_weight)
loaded_params.add(local_name)
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
if weight_name not in local_name:
continue
if any(
@@ -782,11 +791,11 @@ class Step3p5Model(nn.Module):
continue
param = params_dict[replaced_name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(replaced_name)
break
else:
for param_name, weight_name, shard_id in expert_params_mapping:
for param_name, weight_name, expert_shard_id in expert_params_mapping:
if weight_name not in local_name:
continue
replaced_name = local_name.replace(weight_name, param_name)
@@ -823,7 +832,7 @@ class Step3p5Model(nn.Module):
param,
loaded_weight_expert,
replaced_name,
shard_id=shard_id,
shard_id=expert_shard_id,
expert_id=expert_id,
)
loaded_params.add(replaced_name)
@@ -861,10 +870,10 @@ class Step3p5Model(nn.Module):
if local_name not in params_dict:
continue
param = params_dict[local_name]
weight_loader = getattr(
fallback_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
fallback_loader(param, loaded_weight)
loaded_params.add(local_name)
return loaded_params
@@ -918,6 +927,7 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
self.moe_layers.append(layer.moe.experts)
assert len(self.moe_layers) > 0, "No MoE layers found in the model."
assert example_layer is not None
self.num_moe_layers = len(self.moe_layers)
self.num_expert_groups = 1
self.num_shared_experts = 0