diff --git a/tests/models/language/pooling_mteb_test/mteb_embed_utils.py b/tests/models/language/pooling_mteb_test/mteb_embed_utils.py index 101a0a4690..b9deecff8d 100644 --- a/tests/models/language/pooling_mteb_test/mteb_embed_utils.py +++ b/tests/models/language/pooling_mteb_test/mteb_embed_utils.py @@ -163,6 +163,7 @@ def mteb_test_embed_models( hf_model_callback=None, atol=MTEB_EMBED_TOL, prompt_prefix: str | None = None, + vllm_model_callback=None, ): vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs) @@ -178,6 +179,9 @@ def mteb_test_embed_models( ) as vllm_model: model_config = vllm_model.llm.llm_engine.model_config + if vllm_model_callback is not None: + vllm_model_callback(vllm_model) + # Confirm whether vllm is using the correct architecture if model_info.architecture: assert model_info.architecture in model_config.architectures diff --git a/tests/models/language/pooling_mteb_test/test_modernbert_fp8.py b/tests/models/language/pooling_mteb_test/test_modernbert_fp8.py new file mode 100644 index 0000000000..51e9061b9b --- /dev/null +++ b/tests/models/language/pooling_mteb_test/test_modernbert_fp8.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.models.utils import EmbedModelInfo +from tests.quantization.utils import is_quant_method_supported + +from .mteb_embed_utils import mteb_test_embed_models + +MODEL_INFO = EmbedModelInfo( + "Alibaba-NLP/gte-modernbert-base", + mteb_score=0.748193353, + architecture="ModernBertModel", + seq_pooling_type="CLS", + attn_type="encoder_only", + is_prefix_caching_supported=False, + is_chunked_prefill_supported=False, + enable_test=True, +) + + +def _assert_modernbert_online_fp8(model) -> None: + from vllm.model_executor.layers.quantization.online.fp8 import ( + Fp8PerTensorOnlineLinearMethod, + ) + + layer = model.encoder_layer.layers[0] + linears = { + "attn.Wqkv": layer.attn.Wqkv, + "attn.Wo": layer.attn.Wo, + "mlp.Wi": layer.mlp.Wi, + "mlp.Wo": layer.mlp.Wo, + } + for name, linear in linears.items(): + assert isinstance(linear.quant_method, Fp8PerTensorOnlineLinearMethod), ( + f"{name} was not quantized with online FP8" + ) + + +@pytest.mark.skipif( + not is_quant_method_supported("fp8"), + reason="FP8 is not supported on this GPU type.", +) +def test_modernbert_online_fp8_mteb(hf_runner, vllm_runner, monkeypatch) -> None: + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + + def check_online_fp8(vllm_model) -> None: + vllm_model.apply_model(_assert_modernbert_online_fp8) + + mteb_test_embed_models( + hf_runner, + vllm_runner, + MODEL_INFO, + vllm_extra_kwargs={"quantization": "fp8_per_tensor"}, + # Account for Online FP8 kernel variance across GPU architectures. + atol=2e-3, + vllm_model_callback=check_online_fp8, + ) diff --git a/vllm/model_executor/models/modernbert.py b/vllm/model_executor/models/modernbert.py index dca5c9ec45..8fe80bbe19 100644 --- a/vllm/model_executor/models/modernbert.py +++ b/vllm/model_executor/models/modernbert.py @@ -13,7 +13,11 @@ from vllm.distributed import get_tensor_model_parallel_world_size from vllm.model_executor.layers.attention import ( EncoderOnlyAttention, ) -from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) from vllm.model_executor.layers.pooler import DispatchPooler from vllm.model_executor.layers.pooler.activations import LambdaPoolerActivation from vllm.model_executor.layers.pooler.seqwise import ( @@ -22,6 +26,7 @@ from vllm.model_executor.layers.pooler.seqwise import ( get_seq_pooling_method, ) from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_classify +from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding from vllm.model_executor.model_loader.weight_utils import default_weight_loader @@ -68,6 +73,7 @@ class ModernBertAttention(nn.Module): layer_id: int | None = None, prefix: str = "", dtype: torch.dtype | None = None, + quant_config: QuantizationConfig | None = None, ): super().__init__() self.config = config @@ -85,6 +91,7 @@ class ModernBertAttention(nn.Module): self.head_dim, self.num_heads, bias=config.attention_bias, + quant_config=quant_config, prefix=f"{prefix}.Wqkv", ) @@ -128,6 +135,7 @@ class ModernBertAttention(nn.Module): config.hidden_size, config.hidden_size, bias=config.attention_bias, + quant_config=quant_config, prefix=f"{prefix}.Wo", ) @@ -146,22 +154,33 @@ class ModernBertAttention(nn.Module): class ModernBertMLP(nn.Module): - def __init__(self, config: ModernBertConfig, prefix: str = ""): + def __init__( + self, + config: ModernBertConfig, + prefix: str = "", + quant_config: QuantizationConfig | None = None, + ): super().__init__() self.config = config - self.Wi = nn.Linear( - config.hidden_size, int(config.intermediate_size) * 2, bias=config.mlp_bias + self.Wi = MergedColumnParallelLinear( + config.hidden_size, + [int(config.intermediate_size)] * 2, + bias=config.mlp_bias, + quant_config=quant_config, + prefix=f"{prefix}.Wi", ) self.act = ACT2FN[config.hidden_activation] self.Wo = RowParallelLinear( config.intermediate_size, config.hidden_size, bias=config.mlp_bias, + quant_config=quant_config, prefix=f"{prefix}.Wo", ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - input, gate = self.Wi(hidden_states).chunk(2, dim=-1) + input_gate, _ = self.Wi(hidden_states) + input, gate = input_gate.chunk(2, dim=-1) return self.Wo(self.act(input) * gate)[0] @@ -172,6 +191,7 @@ class ModernBertLayer(nn.Module): prefix: str = "", layer_id: int | None = None, dtype: torch.dtype | None = None, + quant_config: QuantizationConfig | None = None, ): super().__init__() self.config = config @@ -186,11 +206,16 @@ class ModernBertLayer(nn.Module): layer_id=layer_id, prefix=f"{prefix}.attn", dtype=dtype, + quant_config=quant_config, ) self.mlp_norm = nn.LayerNorm( config.hidden_size, eps=config.norm_eps, bias=config.norm_bias ) - self.mlp = ModernBertMLP(config, prefix=f"{prefix}.mlp") + self.mlp = ModernBertMLP( + config, + prefix=f"{prefix}.mlp", + quant_config=quant_config, + ) def forward( self, @@ -211,6 +236,7 @@ class ModernBertEncoderLayer(nn.Module): super().__init__() config = vllm_config.model_config.hf_config dtype = vllm_config.model_config.dtype + quant_config = vllm_config.quant_config self.layers = nn.ModuleList( [ ModernBertLayer( @@ -218,6 +244,7 @@ class ModernBertEncoderLayer(nn.Module): layer_id=layer_id, prefix=f"{prefix}.layers.{layer_id}", dtype=dtype, + quant_config=quant_config, ) for layer_id in range(config.num_hidden_layers) ]