[Model] Add FP8 quantization support for ModernBERT (#53101)
Signed-off-by: thunguo <tew@apache.org> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
2ec6f0d71e
commit
e8888b2d68
@@ -163,6 +163,7 @@ def mteb_test_embed_models(
|
||||
hf_model_callback=None,
|
||||
atol=MTEB_EMBED_TOL,
|
||||
prompt_prefix: str | None = None,
|
||||
vllm_model_callback=None,
|
||||
):
|
||||
vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
|
||||
|
||||
@@ -178,6 +179,9 @@ def mteb_test_embed_models(
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
|
||||
if vllm_model_callback is not None:
|
||||
vllm_model_callback(vllm_model)
|
||||
|
||||
# Confirm whether vllm is using the correct architecture
|
||||
if model_info.architecture:
|
||||
assert model_info.architecture in model_config.architectures
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
MODEL_INFO = EmbedModelInfo(
|
||||
"Alibaba-NLP/gte-modernbert-base",
|
||||
mteb_score=0.748193353,
|
||||
architecture="ModernBertModel",
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
)
|
||||
|
||||
|
||||
def _assert_modernbert_online_fp8(model) -> None:
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
|
||||
layer = model.encoder_layer.layers[0]
|
||||
linears = {
|
||||
"attn.Wqkv": layer.attn.Wqkv,
|
||||
"attn.Wo": layer.attn.Wo,
|
||||
"mlp.Wi": layer.mlp.Wi,
|
||||
"mlp.Wo": layer.mlp.Wo,
|
||||
}
|
||||
for name, linear in linears.items():
|
||||
assert isinstance(linear.quant_method, Fp8PerTensorOnlineLinearMethod), (
|
||||
f"{name} was not quantized with online FP8"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_modernbert_online_fp8_mteb(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
def check_online_fp8(vllm_model) -> None:
|
||||
vllm_model.apply_model(_assert_modernbert_online_fp8)
|
||||
|
||||
mteb_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
MODEL_INFO,
|
||||
vllm_extra_kwargs={"quantization": "fp8_per_tensor"},
|
||||
# Account for Online FP8 kernel variance across GPU architectures.
|
||||
atol=2e-3,
|
||||
vllm_model_callback=check_online_fp8,
|
||||
)
|
||||
@@ -13,7 +13,11 @@ from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.attention import (
|
||||
EncoderOnlyAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.model_executor.layers.pooler.activations import LambdaPoolerActivation
|
||||
from vllm.model_executor.layers.pooler.seqwise import (
|
||||
@@ -22,6 +26,7 @@ from vllm.model_executor.layers.pooler.seqwise import (
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_classify
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
@@ -68,6 +73,7 @@ class ModernBertAttention(nn.Module):
|
||||
layer_id: int | None = None,
|
||||
prefix: str = "",
|
||||
dtype: torch.dtype | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -85,6 +91,7 @@ class ModernBertAttention(nn.Module):
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
bias=config.attention_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.Wqkv",
|
||||
)
|
||||
|
||||
@@ -128,6 +135,7 @@ class ModernBertAttention(nn.Module):
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=config.attention_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.Wo",
|
||||
)
|
||||
|
||||
@@ -146,22 +154,33 @@ class ModernBertAttention(nn.Module):
|
||||
|
||||
|
||||
class ModernBertMLP(nn.Module):
|
||||
def __init__(self, config: ModernBertConfig, prefix: str = ""):
|
||||
def __init__(
|
||||
self,
|
||||
config: ModernBertConfig,
|
||||
prefix: str = "",
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.Wi = nn.Linear(
|
||||
config.hidden_size, int(config.intermediate_size) * 2, bias=config.mlp_bias
|
||||
self.Wi = MergedColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
[int(config.intermediate_size)] * 2,
|
||||
bias=config.mlp_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.Wi",
|
||||
)
|
||||
self.act = ACT2FN[config.hidden_activation]
|
||||
self.Wo = RowParallelLinear(
|
||||
config.intermediate_size,
|
||||
config.hidden_size,
|
||||
bias=config.mlp_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.Wo",
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
input, gate = self.Wi(hidden_states).chunk(2, dim=-1)
|
||||
input_gate, _ = self.Wi(hidden_states)
|
||||
input, gate = input_gate.chunk(2, dim=-1)
|
||||
return self.Wo(self.act(input) * gate)[0]
|
||||
|
||||
|
||||
@@ -172,6 +191,7 @@ class ModernBertLayer(nn.Module):
|
||||
prefix: str = "",
|
||||
layer_id: int | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -186,11 +206,16 @@ class ModernBertLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
prefix=f"{prefix}.attn",
|
||||
dtype=dtype,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.mlp_norm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.norm_eps, bias=config.norm_bias
|
||||
)
|
||||
self.mlp = ModernBertMLP(config, prefix=f"{prefix}.mlp")
|
||||
self.mlp = ModernBertMLP(
|
||||
config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -211,6 +236,7 @@ class ModernBertEncoderLayer(nn.Module):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
dtype = vllm_config.model_config.dtype
|
||||
quant_config = vllm_config.quant_config
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
ModernBertLayer(
|
||||
@@ -218,6 +244,7 @@ class ModernBertEncoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
prefix=f"{prefix}.layers.{layer_id}",
|
||||
dtype=dtype,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for layer_id in range(config.num_hidden_layers)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user