[Bugfix] Fix ModernBert load & Enable sliding window attention for bidirectional attention. (#22637)

Signed-off-by: wang.yuqi <noooop@126.com>
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Co-authored-by: Max de Bayser <mbayser@br.ibm.com>
This commit is contained in:
wang.yuqi
2025-08-12 00:23:17 -07:00
committed by GitHub
co-authored by Max de Bayser
parent 2f4657952b
commit 6d729c43fb
4 changed files with 101 additions and 59 deletions
+15 -16
View File
@@ -26,8 +26,7 @@ from vllm.model_executor.pooling_metadata import PoolingMetadata
from vllm.sequence import IntermediateTensors
from vllm.tasks import PoolingTask
from .interfaces import (SupportsCrossEncoding, SupportsV0Only,
default_pooling_type)
from .interfaces import SupportsCrossEncoding, default_pooling_type
from .utils import WeightsMapper, maybe_prefix
@@ -93,16 +92,14 @@ class ModernBertAttention(nn.Module):
bias=config.attention_bias,
)
sliding_window = None
if layer_id % config.global_attn_every_n_layers != 0:
self.local_attention = (config.local_attention // 2,
config.local_attention // 2)
sliding_window = config.local_attention // 2
rope_theta = config.local_rope_theta if config.local_rope_theta \
is not None else config.global_rope_theta
else:
self.local_attention = (-1, -1)
rope_theta = config.global_rope_theta
rope_theta = config.global_rope_theta
if self.local_attention != (
-1, -1) and config.local_rope_theta is not None:
rope_theta = config.local_rope_theta
self.rotary_emb = ModernBertRotaryEmbedding(config=config,
head_size=self.head_dim,
dim=self.head_dim,
@@ -111,7 +108,8 @@ class ModernBertAttention(nn.Module):
self.head_dim,
self.scaling,
prefix=f"{layer_id}.attn",
attn_type=AttentionType.ENCODER_ONLY)
attn_type=AttentionType.ENCODER_ONLY,
per_layer_sliding_window=sliding_window)
self.Wo = RowParallelLinear(config.hidden_size,
config.hidden_size,
bias=config.attention_bias)
@@ -278,6 +276,7 @@ class ModernBertPooler(Pooler):
return self.pooling.get_pooling_updates(task)
def _head(self, pooled_output: torch.Tensor):
pooled_output = pooled_output.to(self.dense.weight.dtype)
return self.norm(self.act(self.dense(pooled_output)))
def forward(
@@ -296,8 +295,7 @@ class ModernBertPooler(Pooler):
@default_pooling_type("CLS")
class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
SupportsCrossEncoding):
class ModernBertForSequenceClassification(nn.Module, SupportsCrossEncoding):
is_pooling_model = True
@@ -308,6 +306,7 @@ class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
self.model = ModernBertModel(vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "modernbert"))
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.pooling = ModernBertPooler(config)
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
@@ -317,14 +316,14 @@ class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
Pooler.for_encode(pooler_config),
"classify":
ClassifierPooler(
pooling=ModernBertPooler(config),
pooling=self.pooling,
classifier=self.classifier,
act_fn=ClassifierPooler.act_fn_for_seq_cls(
vllm_config.model_config),
),
"score":
ClassifierPooler(
pooling=ModernBertPooler(config),
pooling=self.pooling,
classifier=self.classifier,
act_fn=ClassifierPooler.act_fn_for_cross_encoder(
vllm_config.model_config),
@@ -353,7 +352,7 @@ class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
default_weight_loader)
weight_loader(param, loaded_weight)
if name.startswith("head"):
param = params_dict["_pooler.pooler." + name[len("head") + 1:]]
param = params_dict["pooling." + name[len("head") + 1:]]
weight_loader = getattr(param, "weight_loader",
default_weight_loader)
weight_loader(param, loaded_weight)
@@ -368,5 +367,5 @@ class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
return self.model(
input_ids=input_ids,
inputs_embeds=inputs_embeds,
position_ids=positions,
positions=positions,
)