From b6971198008e37637b80b0cdc183bf4bad11376e Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 9 Jun 2026 10:52:36 -0500 Subject: [PATCH] [ROCm][CI] Stabilize ModernBERT token-classification parity against Hugging Face (#44040) Signed-off-by: Andreas Karatzas --- vllm/model_executor/models/modernbert.py | 27 ++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/vllm/model_executor/models/modernbert.py b/vllm/model_executor/models/modernbert.py index a29b1a9fbf..8195f61b05 100644 --- a/vllm/model_executor/models/modernbert.py +++ b/vllm/model_executor/models/modernbert.py @@ -63,7 +63,11 @@ class ModernBertEmbeddings(nn.Module): class ModernBertAttention(nn.Module): def __init__( - self, config: ModernBertConfig, layer_id: int | None = None, prefix: str = "" + self, + config: ModernBertConfig, + layer_id: int | None = None, + prefix: str = "", + dtype: torch.dtype | None = None, ): super().__init__() self.config = config @@ -90,11 +94,13 @@ class ModernBertAttention(nn.Module): rope_parameters = config.rope_parameters[layer_type] sliding_window: int | None = None if layer_type == "sliding_attention": - sliding_window = config.local_attention // 2 + # Treats the local attention boundary as inclusive + sliding_window = config.sliding_window + 1 else: # Transformers v4 sliding_window = None if layer_id % config.global_attn_every_n_layers != 0: + # ModernBertConfig does not expose sliding_window sliding_window = config.local_attention // 2 rope_theta = ( config.local_rope_theta @@ -109,7 +115,7 @@ class ModernBertAttention(nn.Module): head_size=self.head_dim, max_position=config.max_position_embeddings, rope_parameters=rope_parameters, - dtype=torch.float16, + dtype=dtype, ) self.attn = EncoderOnlyAttention( self.num_heads, @@ -146,7 +152,7 @@ class ModernBertMLP(nn.Module): self.Wi = nn.Linear( config.hidden_size, int(config.intermediate_size) * 2, bias=config.mlp_bias ) - self.act = nn.GELU() + self.act = ACT2FN[config.hidden_activation] self.Wo = RowParallelLinear( config.intermediate_size, config.hidden_size, @@ -161,7 +167,11 @@ class ModernBertMLP(nn.Module): class ModernBertLayer(nn.Module): def __init__( - self, config: ModernBertConfig, prefix: str = "", layer_id: int | None = None + self, + config: ModernBertConfig, + prefix: str = "", + layer_id: int | None = None, + dtype: torch.dtype | None = None, ): super().__init__() self.config = config @@ -172,7 +182,10 @@ class ModernBertLayer(nn.Module): config.hidden_size, eps=config.norm_eps, bias=config.norm_bias ) self.attn = ModernBertAttention( - config=config, layer_id=layer_id, prefix=f"{prefix}.attn" + config=config, + layer_id=layer_id, + prefix=f"{prefix}.attn", + dtype=dtype, ) self.mlp_norm = nn.LayerNorm( config.hidden_size, eps=config.norm_eps, bias=config.norm_bias @@ -197,12 +210,14 @@ class ModernBertEncoderLayer(nn.Module): def __init__(self, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config + dtype = vllm_config.model_config.dtype self.layers = nn.ModuleList( [ ModernBertLayer( config=config, layer_id=layer_id, prefix=f"{prefix}.layers.{layer_id}", + dtype=dtype, ) for layer_id in range(config.num_hidden_layers) ]