[Bugfix] Fix ModernBert load & Enable sliding window attention for bidirectional attention. (#22637)
Signed-off-by: wang.yuqi <noooop@126.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Max de Bayser <mbayser@br.ibm.com>
This commit is contained in:
co-authored by
Max de Bayser
parent
2f4657952b
commit
6d729c43fb
@@ -26,8 +26,7 @@ from vllm.model_executor.pooling_metadata import PoolingMetadata
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
from .interfaces import (SupportsCrossEncoding, SupportsV0Only,
|
||||
default_pooling_type)
|
||||
from .interfaces import SupportsCrossEncoding, default_pooling_type
|
||||
from .utils import WeightsMapper, maybe_prefix
|
||||
|
||||
|
||||
@@ -93,16 +92,14 @@ class ModernBertAttention(nn.Module):
|
||||
bias=config.attention_bias,
|
||||
)
|
||||
|
||||
sliding_window = None
|
||||
if layer_id % config.global_attn_every_n_layers != 0:
|
||||
self.local_attention = (config.local_attention // 2,
|
||||
config.local_attention // 2)
|
||||
sliding_window = config.local_attention // 2
|
||||
rope_theta = config.local_rope_theta if config.local_rope_theta \
|
||||
is not None else config.global_rope_theta
|
||||
else:
|
||||
self.local_attention = (-1, -1)
|
||||
rope_theta = config.global_rope_theta
|
||||
|
||||
rope_theta = config.global_rope_theta
|
||||
if self.local_attention != (
|
||||
-1, -1) and config.local_rope_theta is not None:
|
||||
rope_theta = config.local_rope_theta
|
||||
self.rotary_emb = ModernBertRotaryEmbedding(config=config,
|
||||
head_size=self.head_dim,
|
||||
dim=self.head_dim,
|
||||
@@ -111,7 +108,8 @@ class ModernBertAttention(nn.Module):
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
prefix=f"{layer_id}.attn",
|
||||
attn_type=AttentionType.ENCODER_ONLY)
|
||||
attn_type=AttentionType.ENCODER_ONLY,
|
||||
per_layer_sliding_window=sliding_window)
|
||||
self.Wo = RowParallelLinear(config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=config.attention_bias)
|
||||
@@ -278,6 +276,7 @@ class ModernBertPooler(Pooler):
|
||||
return self.pooling.get_pooling_updates(task)
|
||||
|
||||
def _head(self, pooled_output: torch.Tensor):
|
||||
pooled_output = pooled_output.to(self.dense.weight.dtype)
|
||||
return self.norm(self.act(self.dense(pooled_output)))
|
||||
|
||||
def forward(
|
||||
@@ -296,8 +295,7 @@ class ModernBertPooler(Pooler):
|
||||
|
||||
|
||||
@default_pooling_type("CLS")
|
||||
class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
|
||||
SupportsCrossEncoding):
|
||||
class ModernBertForSequenceClassification(nn.Module, SupportsCrossEncoding):
|
||||
|
||||
is_pooling_model = True
|
||||
|
||||
@@ -308,6 +306,7 @@ class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
|
||||
self.model = ModernBertModel(vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "modernbert"))
|
||||
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
|
||||
self.pooling = ModernBertPooler(config)
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
@@ -317,14 +316,14 @@ class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
|
||||
Pooler.for_encode(pooler_config),
|
||||
"classify":
|
||||
ClassifierPooler(
|
||||
pooling=ModernBertPooler(config),
|
||||
pooling=self.pooling,
|
||||
classifier=self.classifier,
|
||||
act_fn=ClassifierPooler.act_fn_for_seq_cls(
|
||||
vllm_config.model_config),
|
||||
),
|
||||
"score":
|
||||
ClassifierPooler(
|
||||
pooling=ModernBertPooler(config),
|
||||
pooling=self.pooling,
|
||||
classifier=self.classifier,
|
||||
act_fn=ClassifierPooler.act_fn_for_cross_encoder(
|
||||
vllm_config.model_config),
|
||||
@@ -353,7 +352,7 @@ class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
|
||||
default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
if name.startswith("head"):
|
||||
param = params_dict["_pooler.pooler." + name[len("head") + 1:]]
|
||||
param = params_dict["pooling." + name[len("head") + 1:]]
|
||||
weight_loader = getattr(param, "weight_loader",
|
||||
default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
@@ -368,5 +367,5 @@ class ModernBertForSequenceClassification(nn.Module, SupportsV0Only,
|
||||
return self.model(
|
||||
input_ids=input_ids,
|
||||
inputs_embeds=inputs_embeds,
|
||||
position_ids=positions,
|
||||
positions=positions,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user