From 5810e884f11875b46fbae0871754d855d8891105 Mon Sep 17 00:00:00 2001 From: Gopala-Krishna Char Date: Wed, 15 Jul 2026 16:30:33 +0200 Subject: [PATCH] [Model] Add RobertaForTokenClassification / XLMRobertaForTokenClassification (#47991) Signed-off-by: krishy91 --- docs/models/pooling_models/token_classify.md | 2 + .../pooling/test_token_classification.py | 40 +++++++++++ tests/models/registry.py | 6 ++ vllm/model_executor/models/registry.py | 5 ++ vllm/model_executor/models/roberta.py | 66 ++++++++++++++++++- 5 files changed, 118 insertions(+), 1 deletion(-) diff --git a/docs/models/pooling_models/token_classify.md b/docs/models/pooling_models/token_classify.md index 6b2cefbde5..faada9c2ce 100644 --- a/docs/models/pooling_models/token_classify.md +++ b/docs/models/pooling_models/token_classify.md @@ -47,6 +47,8 @@ The BAAI/bge-m3 model leverages token classification for sparse retrieval. For m | `ModernBertForTokenClassification` | ModernBERT-based | `disham993/electrical-ner-ModernBERT-base` | | | | `OpenAIPrivacyFilterForTokenClassification` | gpt-oss-based encoder | `openai/privacy-filter` | | | | `Qwen3ForTokenClassification`C | Qwen3-based | `bd2lcco/Qwen3-0.6B-finetuned` | | | +| `RobertaForTokenClassification` | RoBERTa-based | `Jean-Baptiste/roberta-large-ner-english` | | | +| `XLMRobertaForTokenClassification` | XLM-RoBERTa-based | `Davlan/xlm-roberta-base-ner-hrl` | | | | `*Model`C, `*ForCausalLM`C, etc. | Generative models | N/A | \* | \* | C Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion)) diff --git a/tests/models/language/pooling/test_token_classification.py b/tests/models/language/pooling/test_token_classification.py index 2524c7053a..0fdeeea4b3 100644 --- a/tests/models/language/pooling/test_token_classification.py +++ b/tests/models/language/pooling/test_token_classification.py @@ -117,6 +117,46 @@ def test_modernbert_models( torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3) +@pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"]) +@pytest.mark.parametrize("dtype", ["float"]) +@torch.inference_mode +def test_xlm_roberta_models( + hf_runner, + vllm_runner, + example_prompts, + model: str, + dtype: str, +) -> None: + with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model: + vllm_outputs = vllm_model.token_classify(example_prompts) + + # Use eager attention on ROCm to avoid HF Transformers flash attention + # accuracy issues: https://github.com/vllm-project/vllm/issues/30167 + hf_model_kwargs = {} + if current_platform.is_rocm(): + hf_model_kwargs["attn_implementation"] = "eager" + + with hf_runner( + model, + dtype=dtype, + auto_cls=AutoModelForTokenClassification, + model_kwargs=hf_model_kwargs, + ) as hf_model: + tokenizer = hf_model.tokenizer + hf_outputs = [] + for prompt in example_prompts: + inputs = tokenizer([prompt], return_tensors="pt") + inputs = hf_model.wrap_device(inputs) + output = hf_model.model(**inputs) + hf_outputs.append(softmax(output.logits[0])) + + # check logits difference + for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): + hf_output = hf_output.detach().clone().cpu().float() + vllm_output = vllm_output.detach().clone().cpu().float() + torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3) + + PRIVACY_FILTER_PROMPTS = [ "My name is Harry Potter.", "Email me at harry.potter@hogwarts.edu.", diff --git a/tests/models/registry.py b/tests/models/registry.py index 4e8fe96fdb..8d6b45d479 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -713,6 +713,12 @@ _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = { "openai/privacy-filter", min_transformers_version="5.6.0.dev0", ), + "RobertaForTokenClassification": _HfExamplesInfo( + "Jean-Baptiste/roberta-large-ner-english" + ), + "XLMRobertaForTokenClassification": _HfExamplesInfo( + "Davlan/xlm-roberta-base-ner-hrl" + ), } _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = { diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index c8486dac5f..cab03a8a6a 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -297,6 +297,11 @@ _TOKEN_CLASSIFICATION_MODELS = { "qwen3_asr_forced_aligner", "Qwen3ASRForcedAlignerForTokenClassification", ), + "RobertaForTokenClassification": ("roberta", "RobertaForTokenClassification"), + "XLMRobertaForTokenClassification": ( + "roberta", + "RobertaForTokenClassification", + ), } _SEQUENCE_CLASSIFICATION_MODELS = { diff --git a/vllm/model_executor/models/roberta.py b/vllm/model_executor/models/roberta.py index a2419fd1c6..948a939b9d 100644 --- a/vllm/model_executor/models/roberta.py +++ b/vllm/model_executor/models/roberta.py @@ -42,7 +42,7 @@ from vllm.sequence import IntermediateTensors from .bert_with_rope import BertWithRope, JinaRobertaModel from .interfaces import SupportsCrossEncoding -from .interfaces_base import default_pooling_type +from .interfaces_base import attn_type, default_pooling_type class RobertaEmbedding(nn.Module): @@ -332,3 +332,67 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding): inputs_embeds=inputs_embeds, intermediate_tensors=intermediate_tensors, ) + + +@attn_type("encoder_only") +@default_pooling_type(tok_pooling_type="ALL") +class RobertaForTokenClassification(nn.Module): + """A model that uses Roberta to provide token classification. + + Mirrors BertForTokenClassification, swapping in RobertaEmbedding for the + RoBERTa/XLM-RoBERTa position-embedding offset and weight layout. + + Also registered as XLMRobertaForTokenClassification since XLM-RoBERTa + checkpoints share RoBERTa's architecture and `roberta.*` weight prefix. + """ + + is_pooling_model = True + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + self.head_dtype = vllm_config.model_config.head_dtype + self.num_labels = config.num_labels + self.roberta = BertModel( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "roberta"), + embedding_class=RobertaEmbedding, + ) + self.classifier = nn.Linear( + config.hidden_size, config.num_labels, dtype=self.head_dtype + ) + + pooler_config = vllm_config.model_config.pooler_config + assert pooler_config is not None + + self.pooler = pooler_for_token_classify(pooler_config) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.roberta.embed_input_ids(input_ids) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + loader = AutoWeightsLoader(self) + return loader.load_weights(weights) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + token_type_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + if token_type_ids is not None: + assert self.roberta.config.vocab_size < (1 << TOKEN_TYPE_SHIFT) + assert input_ids is not None + _encode_token_type_ids(input_ids, token_type_ids) + + hidden_states = self.roberta( + input_ids=input_ids, + positions=positions, + inputs_embeds=inputs_embeds, + intermediate_tensors=intermediate_tensors, + ) + + hidden_states = hidden_states.to(self.head_dtype) + return self.classifier(hidden_states)