[ROCm][CI] Stabilize ModernBERT token-classification parity against Hugging Face (#44040)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-09 16:52:36 +01:00
committed by GitHub
parent b4c6dc6454
commit b697119800
+21 -6
View File
@@ -63,7 +63,11 @@ class ModernBertEmbeddings(nn.Module):
class ModernBertAttention(nn.Module):
def __init__(
self, config: ModernBertConfig, layer_id: int | None = None, prefix: str = ""
self,
config: ModernBertConfig,
layer_id: int | None = None,
prefix: str = "",
dtype: torch.dtype | None = None,
):
super().__init__()
self.config = config
@@ -90,11 +94,13 @@ class ModernBertAttention(nn.Module):
rope_parameters = config.rope_parameters[layer_type]
sliding_window: int | None = None
if layer_type == "sliding_attention":
sliding_window = config.local_attention // 2
# Treats the local attention boundary as inclusive
sliding_window = config.sliding_window + 1
else:
# Transformers v4
sliding_window = None
if layer_id % config.global_attn_every_n_layers != 0:
# ModernBertConfig does not expose sliding_window
sliding_window = config.local_attention // 2
rope_theta = (
config.local_rope_theta
@@ -109,7 +115,7 @@ class ModernBertAttention(nn.Module):
head_size=self.head_dim,
max_position=config.max_position_embeddings,
rope_parameters=rope_parameters,
dtype=torch.float16,
dtype=dtype,
)
self.attn = EncoderOnlyAttention(
self.num_heads,
@@ -146,7 +152,7 @@ class ModernBertMLP(nn.Module):
self.Wi = nn.Linear(
config.hidden_size, int(config.intermediate_size) * 2, bias=config.mlp_bias
)
self.act = nn.GELU()
self.act = ACT2FN[config.hidden_activation]
self.Wo = RowParallelLinear(
config.intermediate_size,
config.hidden_size,
@@ -161,7 +167,11 @@ class ModernBertMLP(nn.Module):
class ModernBertLayer(nn.Module):
def __init__(
self, config: ModernBertConfig, prefix: str = "", layer_id: int | None = None
self,
config: ModernBertConfig,
prefix: str = "",
layer_id: int | None = None,
dtype: torch.dtype | None = None,
):
super().__init__()
self.config = config
@@ -172,7 +182,10 @@ class ModernBertLayer(nn.Module):
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
)
self.attn = ModernBertAttention(
config=config, layer_id=layer_id, prefix=f"{prefix}.attn"
config=config,
layer_id=layer_id,
prefix=f"{prefix}.attn",
dtype=dtype,
)
self.mlp_norm = nn.LayerNorm(
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
@@ -197,12 +210,14 @@ class ModernBertEncoderLayer(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
dtype = vllm_config.model_config.dtype
self.layers = nn.ModuleList(
[
ModernBertLayer(
config=config,
layer_id=layer_id,
prefix=f"{prefix}.layers.{layer_id}",
dtype=dtype,
)
for layer_id in range(config.num_hidden_layers)
]