[ROCm][CI] Stabilize ModernBERT token-classification parity against Hugging Face (#44040)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -63,7 +63,11 @@ class ModernBertEmbeddings(nn.Module):
|
||||
|
||||
class ModernBertAttention(nn.Module):
|
||||
def __init__(
|
||||
self, config: ModernBertConfig, layer_id: int | None = None, prefix: str = ""
|
||||
self,
|
||||
config: ModernBertConfig,
|
||||
layer_id: int | None = None,
|
||||
prefix: str = "",
|
||||
dtype: torch.dtype | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -90,11 +94,13 @@ class ModernBertAttention(nn.Module):
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
sliding_window: int | None = None
|
||||
if layer_type == "sliding_attention":
|
||||
sliding_window = config.local_attention // 2
|
||||
# Treats the local attention boundary as inclusive
|
||||
sliding_window = config.sliding_window + 1
|
||||
else:
|
||||
# Transformers v4
|
||||
sliding_window = None
|
||||
if layer_id % config.global_attn_every_n_layers != 0:
|
||||
# ModernBertConfig does not expose sliding_window
|
||||
sliding_window = config.local_attention // 2
|
||||
rope_theta = (
|
||||
config.local_rope_theta
|
||||
@@ -109,7 +115,7 @@ class ModernBertAttention(nn.Module):
|
||||
head_size=self.head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
rope_parameters=rope_parameters,
|
||||
dtype=torch.float16,
|
||||
dtype=dtype,
|
||||
)
|
||||
self.attn = EncoderOnlyAttention(
|
||||
self.num_heads,
|
||||
@@ -146,7 +152,7 @@ class ModernBertMLP(nn.Module):
|
||||
self.Wi = nn.Linear(
|
||||
config.hidden_size, int(config.intermediate_size) * 2, bias=config.mlp_bias
|
||||
)
|
||||
self.act = nn.GELU()
|
||||
self.act = ACT2FN[config.hidden_activation]
|
||||
self.Wo = RowParallelLinear(
|
||||
config.intermediate_size,
|
||||
config.hidden_size,
|
||||
@@ -161,7 +167,11 @@ class ModernBertMLP(nn.Module):
|
||||
|
||||
class ModernBertLayer(nn.Module):
|
||||
def __init__(
|
||||
self, config: ModernBertConfig, prefix: str = "", layer_id: int | None = None
|
||||
self,
|
||||
config: ModernBertConfig,
|
||||
prefix: str = "",
|
||||
layer_id: int | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -172,7 +182,10 @@ class ModernBertLayer(nn.Module):
|
||||
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
|
||||
)
|
||||
self.attn = ModernBertAttention(
|
||||
config=config, layer_id=layer_id, prefix=f"{prefix}.attn"
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
prefix=f"{prefix}.attn",
|
||||
dtype=dtype,
|
||||
)
|
||||
self.mlp_norm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
|
||||
@@ -197,12 +210,14 @@ class ModernBertEncoderLayer(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
dtype = vllm_config.model_config.dtype
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
ModernBertLayer(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
prefix=f"{prefix}.layers.{layer_id}",
|
||||
dtype=dtype,
|
||||
)
|
||||
for layer_id in range(config.num_hidden_layers)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user