From 7738ef35b8c62070773ef3cfd53f2a59d94626b1 Mon Sep 17 00:00:00 2001 From: Snehlata Date: Tue, 14 Jul 2026 02:26:25 +0530 Subject: [PATCH] [Feat] Add Support for BertForMaskedLM to vLLM (#48463) Signed-off-by: atalhens --- .../pooling/test_token_classification.py | 55 +++++++++- tests/models/registry.py | 1 + vllm/model_executor/models/bert.py | 103 +++++++++++++++++- vllm/model_executor/models/registry.py | 1 + 4 files changed, 158 insertions(+), 2 deletions(-) diff --git a/tests/models/language/pooling/test_token_classification.py b/tests/models/language/pooling/test_token_classification.py index 09652885d1..2524c7053a 100644 --- a/tests/models/language/pooling/test_token_classification.py +++ b/tests/models/language/pooling/test_token_classification.py @@ -3,7 +3,7 @@ import pytest import torch -from transformers import AutoModelForTokenClassification +from transformers import AutoModelForMaskedLM, AutoModelForTokenClassification from tests.models.registry import HF_EXAMPLE_MODELS from tests.models.utils import softmax @@ -200,3 +200,56 @@ def test_auto_conversion( hf_output = hf_output.detach().clone().cpu().float() vllm_output = vllm_output.detach().clone().cpu().float() assert torch.allclose(hf_output, vllm_output, atol=1e-2) + + +@pytest.mark.parametrize( + "model", + [ + # Original Google checkpoint: legacy `gamma`/`beta` LayerNorm names, an + # NSP head (`cls.seq_relationship.*`) and a decoder tied to the input + # embeddings (no explicit decoder weight in the checkpoint). + "google-bert/bert-base-uncased", + ], +) +@pytest.mark.parametrize("dtype", ["float"]) +@torch.inference_mode +def test_bert_for_masked_lm( + hf_runner, + vllm_runner, + example_prompts, + model: str, + dtype: str, +) -> None: + # BertForMaskedLM exposes its MLM head as a token-level pooling task; the + # head applies softmax over the vocabulary, so each output row is a + # distribution (matching HF's softmax(logits) below). + with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model: + vllm_outputs = vllm_model.token_classify(example_prompts) + + # Use eager attention on ROCm to avoid HF Transformers flash attention + # accuracy issues: https://github.com/vllm-project/vllm/issues/30167 + hf_model_kwargs = {} + if current_platform.is_rocm(): + hf_model_kwargs["attn_implementation"] = "eager" + + with hf_runner( + model, + dtype=dtype, + auto_cls=AutoModelForMaskedLM, + model_kwargs=hf_model_kwargs, + ) as hf_model: + tokenizer = hf_model.tokenizer + hf_outputs = [] + for prompt in example_prompts: + inputs = tokenizer([prompt], return_tensors="pt") + inputs = hf_model.wrap_device(inputs) + output = hf_model.model(**inputs) + hf_outputs.append(softmax(output.logits[0])) + + # Compare the per-token vocabulary distributions position by position. + for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): + hf_output = hf_output.detach().clone().cpu().float() + vllm_output = vllm_output.detach().clone().cpu().float() + assert hf_output.shape == vllm_output.shape + assert torch.equal(hf_output.argmax(dim=-1), vllm_output.argmax(dim=-1)) + torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3) diff --git a/tests/models/registry.py b/tests/models/registry.py index 7596501645..4157d23984 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -576,6 +576,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { _EMBEDDING_EXAMPLE_MODELS = { # [Text-only] "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"), + "BertForMaskedLM": _HfExamplesInfo("google-bert/bert-base-uncased"), "BertSpladeSparseEmbeddingModel": _HfExamplesInfo( "naver/splade-v3", hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]}, diff --git a/vllm/model_executor/models/bert.py b/vllm/model_executor/models/bert.py index cf968902f6..225f6c2bda 100644 --- a/vllm/model_executor/models/bert.py +++ b/vllm/model_executor/models/bert.py @@ -370,11 +370,17 @@ class BertModel(nn.Module, SupportsQuant): packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]} hf_to_vllm_mapper = WeightsMapper( + # Original google-bert checkpoints use the legacy `gamma`/`beta` + # LayerNorm names; rename to vLLM's `weight`/`bias`. + orig_to_new_substr={ + "LayerNorm.gamma": "LayerNorm.weight", + "LayerNorm.beta": "LayerNorm.bias", + }, orig_to_new_stacked={ ".self.query": (".self.qkv_proj", "q"), ".self.key": (".self.qkv_proj", "k"), ".self.value": (".self.qkv_proj", "v"), - } + }, ) def __init__( @@ -877,3 +883,98 @@ class BertForTokenClassification(nn.Module): hidden_states = hidden_states.to(self.head_dtype) return self.classifier(hidden_states) + + +@attn_type("encoder_only") +@default_pooling_type(tok_pooling_type="ALL") +class BertForMaskedLM(nn.Module): + """Bert with a masked-language-modeling head on top of ``BertModel``. + + Produces per-token logits over the vocabulary. In vLLM terms this is a + token-level pooling model (``tok_pooling_type="ALL"``): the encoder output + is projected by the MLM head to ``vocab_size`` logits for every position, + and the token pooler returns one vector per token. + """ + + is_pooling_model = True + + # Map the HF ``cls.predictions.*`` checkpoint names onto our ``mlm_head.*`` + # submodule. Order matters: the ``None`` (drop) rules and the more specific + # names are listed before the broader ``cls.predictions.decoder`` rule so + # that substring replacement doesn't rewrite them first. + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={ + # Next-sentence-prediction head: not part of masked LM. + "cls.seq_relationship": None, + # Some checkpoints ship an explicit (tied) decoder bias; we load the + # canonical ``cls.predictions.bias`` instead, so drop the duplicate. + "cls.predictions.decoder.bias": None, + # Legacy LayerNorm affine names in the MLM head transform. + "cls.predictions.transform.LayerNorm.gamma": "mlm_head.layer_norm.weight", + "cls.predictions.transform.LayerNorm.beta": "mlm_head.layer_norm.bias", + "cls.predictions.transform.LayerNorm": "mlm_head.layer_norm", + "cls.predictions.transform.dense": "mlm_head.dense", + "cls.predictions.decoder": "mlm_head.decoder", + # In HF ``cls.predictions.bias`` *is* the decoder bias. + "cls.predictions.bias": "mlm_head.decoder.bias", + } + ) + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + self.bert = BertModel( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "bert"), + embedding_class=BertEmbedding, + ) + self.mlm_head = BertMLMHead( + hidden_size=config.hidden_size, + vocab_size=config.vocab_size, + layer_norm_eps=getattr(config, "layer_norm_eps", 1e-12), + ) + + pooler_config = vllm_config.model_config.pooler_config + assert pooler_config is not None + + self.pooler = pooler_for_token_classify(pooler_config) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.bert.embed_input_ids(input_ids) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + + # The MLM decoder shares its weight with the input embeddings. When the + # checkpoint relies on `tie_word_embeddings` (e.g. google-bert/*) it + # doesn't ship an explicit decoder weight, so tie it here. Copy the + # first `vocab_size` rows since VocabParallelEmbedding may pad the vocab. + if "mlm_head.decoder.weight" not in loaded: + emb = self.bert.embeddings.word_embeddings.weight + decoder = self.mlm_head.decoder.weight + decoder.data.copy_(emb.data[: decoder.shape[0]]) + loaded.add("mlm_head.decoder.weight") + return loaded + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + token_type_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + if token_type_ids is not None: + assert self.bert.config.vocab_size < (1 << TOKEN_TYPE_SHIFT) + assert input_ids is not None + _encode_token_type_ids(input_ids, token_type_ids) + + hidden_states = self.bert( + input_ids=input_ids, + positions=positions, + inputs_embeds=inputs_embeds, + intermediate_tensors=intermediate_tensors, + ) + + return self.mlm_head(hidden_states) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 4999662dd6..807c0dc40b 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -210,6 +210,7 @@ _TEXT_GENERATION_MODELS = { _EMBEDDING_MODELS = { # [Text-only] "BertModel": ("bert", "BertEmbeddingModel"), + "BertForMaskedLM": ("bert", "BertForMaskedLM"), "BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"), "BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"), "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),