[Feat] Add Support for BertForMaskedLM to vLLM (#48463)
Signed-off-by: atalhens <sneh.lata@nutanix.com>
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForTokenClassification
|
||||
from transformers import AutoModelForMaskedLM, AutoModelForTokenClassification
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from tests.models.utils import softmax
|
||||
@@ -200,3 +200,56 @@ def test_auto_conversion(
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
# Original Google checkpoint: legacy `gamma`/`beta` LayerNorm names, an
|
||||
# NSP head (`cls.seq_relationship.*`) and a decoder tied to the input
|
||||
# embeddings (no explicit decoder weight in the checkpoint).
|
||||
"google-bert/bert-base-uncased",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@torch.inference_mode
|
||||
def test_bert_for_masked_lm(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# BertForMaskedLM exposes its MLM head as a token-level pooling task; the
|
||||
# head applies softmax over the vocabulary, so each output row is a
|
||||
# distribution (matching HF's softmax(logits) below).
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
# Use eager attention on ROCm to avoid HF Transformers flash attention
|
||||
# accuracy issues: https://github.com/vllm-project/vllm/issues/30167
|
||||
hf_model_kwargs = {}
|
||||
if current_platform.is_rocm():
|
||||
hf_model_kwargs["attn_implementation"] = "eager"
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForMaskedLM,
|
||||
model_kwargs=hf_model_kwargs,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
# Compare the per-token vocabulary distributions position by position.
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
assert hf_output.shape == vllm_output.shape
|
||||
assert torch.equal(hf_output.argmax(dim=-1), vllm_output.argmax(dim=-1))
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
|
||||
@@ -576,6 +576,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
_EMBEDDING_EXAMPLE_MODELS = {
|
||||
# [Text-only]
|
||||
"BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
|
||||
"BertForMaskedLM": _HfExamplesInfo("google-bert/bert-base-uncased"),
|
||||
"BertSpladeSparseEmbeddingModel": _HfExamplesInfo(
|
||||
"naver/splade-v3",
|
||||
hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]},
|
||||
|
||||
@@ -370,11 +370,17 @@ class BertModel(nn.Module, SupportsQuant):
|
||||
packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
# Original google-bert checkpoints use the legacy `gamma`/`beta`
|
||||
# LayerNorm names; rename to vLLM's `weight`/`bias`.
|
||||
orig_to_new_substr={
|
||||
"LayerNorm.gamma": "LayerNorm.weight",
|
||||
"LayerNorm.beta": "LayerNorm.bias",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".self.query": (".self.qkv_proj", "q"),
|
||||
".self.key": (".self.qkv_proj", "k"),
|
||||
".self.value": (".self.qkv_proj", "v"),
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -877,3 +883,98 @@ class BertForTokenClassification(nn.Module):
|
||||
|
||||
hidden_states = hidden_states.to(self.head_dtype)
|
||||
return self.classifier(hidden_states)
|
||||
|
||||
|
||||
@attn_type("encoder_only")
|
||||
@default_pooling_type(tok_pooling_type="ALL")
|
||||
class BertForMaskedLM(nn.Module):
|
||||
"""Bert with a masked-language-modeling head on top of ``BertModel``.
|
||||
|
||||
Produces per-token logits over the vocabulary. In vLLM terms this is a
|
||||
token-level pooling model (``tok_pooling_type="ALL"``): the encoder output
|
||||
is projected by the MLM head to ``vocab_size`` logits for every position,
|
||||
and the token pooler returns one vector per token.
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
|
||||
# Map the HF ``cls.predictions.*`` checkpoint names onto our ``mlm_head.*``
|
||||
# submodule. Order matters: the ``None`` (drop) rules and the more specific
|
||||
# names are listed before the broader ``cls.predictions.decoder`` rule so
|
||||
# that substring replacement doesn't rewrite them first.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
# Next-sentence-prediction head: not part of masked LM.
|
||||
"cls.seq_relationship": None,
|
||||
# Some checkpoints ship an explicit (tied) decoder bias; we load the
|
||||
# canonical ``cls.predictions.bias`` instead, so drop the duplicate.
|
||||
"cls.predictions.decoder.bias": None,
|
||||
# Legacy LayerNorm affine names in the MLM head transform.
|
||||
"cls.predictions.transform.LayerNorm.gamma": "mlm_head.layer_norm.weight",
|
||||
"cls.predictions.transform.LayerNorm.beta": "mlm_head.layer_norm.bias",
|
||||
"cls.predictions.transform.LayerNorm": "mlm_head.layer_norm",
|
||||
"cls.predictions.transform.dense": "mlm_head.dense",
|
||||
"cls.predictions.decoder": "mlm_head.decoder",
|
||||
# In HF ``cls.predictions.bias`` *is* the decoder bias.
|
||||
"cls.predictions.bias": "mlm_head.decoder.bias",
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.bert = BertModel(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "bert"),
|
||||
embedding_class=BertEmbedding,
|
||||
)
|
||||
self.mlm_head = BertMLMHead(
|
||||
hidden_size=config.hidden_size,
|
||||
vocab_size=config.vocab_size,
|
||||
layer_norm_eps=getattr(config, "layer_norm_eps", 1e-12),
|
||||
)
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
self.pooler = pooler_for_token_classify(pooler_config)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.bert.embed_input_ids(input_ids)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
# The MLM decoder shares its weight with the input embeddings. When the
|
||||
# checkpoint relies on `tie_word_embeddings` (e.g. google-bert/*) it
|
||||
# doesn't ship an explicit decoder weight, so tie it here. Copy the
|
||||
# first `vocab_size` rows since VocabParallelEmbedding may pad the vocab.
|
||||
if "mlm_head.decoder.weight" not in loaded:
|
||||
emb = self.bert.embeddings.word_embeddings.weight
|
||||
decoder = self.mlm_head.decoder.weight
|
||||
decoder.data.copy_(emb.data[: decoder.shape[0]])
|
||||
loaded.add("mlm_head.decoder.weight")
|
||||
return loaded
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
token_type_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if token_type_ids is not None:
|
||||
assert self.bert.config.vocab_size < (1 << TOKEN_TYPE_SHIFT)
|
||||
assert input_ids is not None
|
||||
_encode_token_type_ids(input_ids, token_type_ids)
|
||||
|
||||
hidden_states = self.bert(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
inputs_embeds=inputs_embeds,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
)
|
||||
|
||||
return self.mlm_head(hidden_states)
|
||||
|
||||
@@ -210,6 +210,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
_EMBEDDING_MODELS = {
|
||||
# [Text-only]
|
||||
"BertModel": ("bert", "BertEmbeddingModel"),
|
||||
"BertForMaskedLM": ("bert", "BertForMaskedLM"),
|
||||
"BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
|
||||
"BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
|
||||
"DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
|
||||
|
||||
Reference in New Issue
Block a user