[Misc][Model][Refactor] Pass the prefix into Linear layers (#31669)
Signed-off-by: Wang Kunpeng <1289706727@qq.com>
This commit is contained in:
@@ -63,7 +63,9 @@ class ModernBertEmbeddings(nn.Module):
|
||||
|
||||
|
||||
class ModernBertAttention(nn.Module):
|
||||
def __init__(self, config: ModernBertConfig, layer_id: int | None = None):
|
||||
def __init__(
|
||||
self, config: ModernBertConfig, layer_id: int | None = None, prefix: str = ""
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
@@ -80,6 +82,7 @@ class ModernBertAttention(nn.Module):
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
bias=config.attention_bias,
|
||||
prefix=f"{prefix}.Wqkv",
|
||||
)
|
||||
|
||||
if layer_types := getattr(config, "layer_types", None):
|
||||
@@ -117,7 +120,10 @@ class ModernBertAttention(nn.Module):
|
||||
per_layer_sliding_window=sliding_window,
|
||||
)
|
||||
self.Wo = RowParallelLinear(
|
||||
config.hidden_size, config.hidden_size, bias=config.attention_bias
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=config.attention_bias,
|
||||
prefix=f"{prefix}.Wo",
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -135,7 +141,7 @@ class ModernBertAttention(nn.Module):
|
||||
|
||||
|
||||
class ModernBertMLP(nn.Module):
|
||||
def __init__(self, config: ModernBertConfig):
|
||||
def __init__(self, config: ModernBertConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.Wi = nn.Linear(
|
||||
@@ -143,7 +149,10 @@ class ModernBertMLP(nn.Module):
|
||||
)
|
||||
self.act = nn.GELU()
|
||||
self.Wo = RowParallelLinear(
|
||||
config.intermediate_size, config.hidden_size, bias=config.mlp_bias
|
||||
config.intermediate_size,
|
||||
config.hidden_size,
|
||||
bias=config.mlp_bias,
|
||||
prefix=f"{prefix}.Wo",
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -163,11 +172,13 @@ class ModernBertLayer(nn.Module):
|
||||
self.attn_norm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
|
||||
)
|
||||
self.attn = ModernBertAttention(config=config, layer_id=layer_id)
|
||||
self.attn = ModernBertAttention(
|
||||
config=config, layer_id=layer_id, prefix=f"{prefix}.attn"
|
||||
)
|
||||
self.mlp_norm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
|
||||
)
|
||||
self.mlp = ModernBertMLP(config)
|
||||
self.mlp = ModernBertMLP(config, prefix=f"{prefix}.mlp")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -189,7 +200,11 @@ class ModernBertEncoderLayer(nn.Module):
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
ModernBertLayer(config=config, layer_id=layer_id)
|
||||
ModernBertLayer(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
prefix=f"{prefix}.layers.{layer_id}",
|
||||
)
|
||||
for layer_id in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
@@ -220,7 +235,9 @@ class ModernBertModel(nn.Module):
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
self.embeddings = ModernBertEmbeddings(config)
|
||||
self.encoder_layer = ModernBertEncoderLayer(vllm_config)
|
||||
self.encoder_layer = ModernBertEncoderLayer(
|
||||
vllm_config, prefix=f"{prefix}.encoder_layer"
|
||||
)
|
||||
self.final_norm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user