[Model] Add RobertaForTokenClassification / XLMRobertaForTokenClassification (#47991)
Signed-off-by: krishy91 <crgkc.r@gmail.com>
This commit is contained in:
@@ -47,6 +47,8 @@ The BAAI/bge-m3 model leverages token classification for sparse retrieval. For m
|
||||
| `ModernBertForTokenClassification` | ModernBERT-based | `disham993/electrical-ner-ModernBERT-base` | | |
|
||||
| `OpenAIPrivacyFilterForTokenClassification` | gpt-oss-based encoder | `openai/privacy-filter` | | |
|
||||
| `Qwen3ForTokenClassification`<sup>C</sup> | Qwen3-based | `bd2lcco/Qwen3-0.6B-finetuned` | | |
|
||||
| `RobertaForTokenClassification` | RoBERTa-based | `Jean-Baptiste/roberta-large-ner-english` | | |
|
||||
| `XLMRobertaForTokenClassification` | XLM-RoBERTa-based | `Davlan/xlm-roberta-base-ner-hrl` | | |
|
||||
| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | \* | \* |
|
||||
|
||||
<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))
|
||||
|
||||
@@ -117,6 +117,46 @@ def test_modernbert_models(
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"])
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@torch.inference_mode
|
||||
def test_xlm_roberta_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
# Use eager attention on ROCm to avoid HF Transformers flash attention
|
||||
# accuracy issues: https://github.com/vllm-project/vllm/issues/30167
|
||||
hf_model_kwargs = {}
|
||||
if current_platform.is_rocm():
|
||||
hf_model_kwargs["attn_implementation"] = "eager"
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForTokenClassification,
|
||||
model_kwargs=hf_model_kwargs,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
|
||||
|
||||
PRIVACY_FILTER_PROMPTS = [
|
||||
"My name is Harry Potter.",
|
||||
"Email me at harry.potter@hogwarts.edu.",
|
||||
|
||||
@@ -713,6 +713,12 @@ _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = {
|
||||
"openai/privacy-filter",
|
||||
min_transformers_version="5.6.0.dev0",
|
||||
),
|
||||
"RobertaForTokenClassification": _HfExamplesInfo(
|
||||
"Jean-Baptiste/roberta-large-ner-english"
|
||||
),
|
||||
"XLMRobertaForTokenClassification": _HfExamplesInfo(
|
||||
"Davlan/xlm-roberta-base-ner-hrl"
|
||||
),
|
||||
}
|
||||
|
||||
_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
|
||||
|
||||
@@ -297,6 +297,11 @@ _TOKEN_CLASSIFICATION_MODELS = {
|
||||
"qwen3_asr_forced_aligner",
|
||||
"Qwen3ASRForcedAlignerForTokenClassification",
|
||||
),
|
||||
"RobertaForTokenClassification": ("roberta", "RobertaForTokenClassification"),
|
||||
"XLMRobertaForTokenClassification": (
|
||||
"roberta",
|
||||
"RobertaForTokenClassification",
|
||||
),
|
||||
}
|
||||
|
||||
_SEQUENCE_CLASSIFICATION_MODELS = {
|
||||
|
||||
@@ -42,7 +42,7 @@ from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .bert_with_rope import BertWithRope, JinaRobertaModel
|
||||
from .interfaces import SupportsCrossEncoding
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .interfaces_base import attn_type, default_pooling_type
|
||||
|
||||
|
||||
class RobertaEmbedding(nn.Module):
|
||||
@@ -332,3 +332,67 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
|
||||
inputs_embeds=inputs_embeds,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
)
|
||||
|
||||
|
||||
@attn_type("encoder_only")
|
||||
@default_pooling_type(tok_pooling_type="ALL")
|
||||
class RobertaForTokenClassification(nn.Module):
|
||||
"""A model that uses Roberta to provide token classification.
|
||||
|
||||
Mirrors BertForTokenClassification, swapping in RobertaEmbedding for the
|
||||
RoBERTa/XLM-RoBERTa position-embedding offset and weight layout.
|
||||
|
||||
Also registered as XLMRobertaForTokenClassification since XLM-RoBERTa
|
||||
checkpoints share RoBERTa's architecture and `roberta.*` weight prefix.
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.head_dtype = vllm_config.model_config.head_dtype
|
||||
self.num_labels = config.num_labels
|
||||
self.roberta = BertModel(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "roberta"),
|
||||
embedding_class=RobertaEmbedding,
|
||||
)
|
||||
self.classifier = nn.Linear(
|
||||
config.hidden_size, config.num_labels, dtype=self.head_dtype
|
||||
)
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
self.pooler = pooler_for_token_classify(pooler_config)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.roberta.embed_input_ids(input_ids)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
token_type_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if token_type_ids is not None:
|
||||
assert self.roberta.config.vocab_size < (1 << TOKEN_TYPE_SHIFT)
|
||||
assert input_ids is not None
|
||||
_encode_token_type_ids(input_ids, token_type_ids)
|
||||
|
||||
hidden_states = self.roberta(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
inputs_embeds=inputs_embeds,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
)
|
||||
|
||||
hidden_states = hidden_states.to(self.head_dtype)
|
||||
return self.classifier(hidden_states)
|
||||
|
||||
Reference in New Issue
Block a user