[Misc][Model][Refactor] Pass the prefix into Linear layers (#31669)

Signed-off-by: Wang Kunpeng <1289706727@qq.com>
This commit is contained in:
Wang Kunpeng
2026-01-05 20:03:18 +00:00
committed by GitHub
parent 02dbb933cb
commit 5708297e4e
17 changed files with 181 additions and 40 deletions
+25 -8
View File
@@ -63,7 +63,9 @@ class ModernBertEmbeddings(nn.Module):
class ModernBertAttention(nn.Module):
def __init__(self, config: ModernBertConfig, layer_id: int | None = None):
def __init__(
self, config: ModernBertConfig, layer_id: int | None = None, prefix: str = ""
):
super().__init__()
self.config = config
self.hidden_size = config.hidden_size
@@ -80,6 +82,7 @@ class ModernBertAttention(nn.Module):
self.head_dim,
self.num_heads,
bias=config.attention_bias,
prefix=f"{prefix}.Wqkv",
)
if layer_types := getattr(config, "layer_types", None):
@@ -117,7 +120,10 @@ class ModernBertAttention(nn.Module):
per_layer_sliding_window=sliding_window,
)
self.Wo = RowParallelLinear(
config.hidden_size, config.hidden_size, bias=config.attention_bias
config.hidden_size,
config.hidden_size,
bias=config.attention_bias,
prefix=f"{prefix}.Wo",
)
def forward(
@@ -135,7 +141,7 @@ class ModernBertAttention(nn.Module):
class ModernBertMLP(nn.Module):
def __init__(self, config: ModernBertConfig):
def __init__(self, config: ModernBertConfig, prefix: str = ""):
super().__init__()
self.config = config
self.Wi = nn.Linear(
@@ -143,7 +149,10 @@ class ModernBertMLP(nn.Module):
)
self.act = nn.GELU()
self.Wo = RowParallelLinear(
config.intermediate_size, config.hidden_size, bias=config.mlp_bias
config.intermediate_size,
config.hidden_size,
bias=config.mlp_bias,
prefix=f"{prefix}.Wo",
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -163,11 +172,13 @@ class ModernBertLayer(nn.Module):
self.attn_norm = nn.LayerNorm(
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
)
self.attn = ModernBertAttention(config=config, layer_id=layer_id)
self.attn = ModernBertAttention(
config=config, layer_id=layer_id, prefix=f"{prefix}.attn"
)
self.mlp_norm = nn.LayerNorm(
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
)
self.mlp = ModernBertMLP(config)
self.mlp = ModernBertMLP(config, prefix=f"{prefix}.mlp")
def forward(
self,
@@ -189,7 +200,11 @@ class ModernBertEncoderLayer(nn.Module):
config = vllm_config.model_config.hf_config
self.layers = nn.ModuleList(
[
ModernBertLayer(config=config, layer_id=layer_id)
ModernBertLayer(
config=config,
layer_id=layer_id,
prefix=f"{prefix}.layers.{layer_id}",
)
for layer_id in range(config.num_hidden_layers)
]
)
@@ -220,7 +235,9 @@ class ModernBertModel(nn.Module):
config = vllm_config.model_config.hf_config
self.config = config
self.embeddings = ModernBertEmbeddings(config)
self.encoder_layer = ModernBertEncoderLayer(vllm_config)
self.encoder_layer = ModernBertEncoderLayer(
vllm_config, prefix=f"{prefix}.encoder_layer"
)
self.final_norm = nn.LayerNorm(
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
)