[Model Runner V2] Enable encoder token classification (#50293)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-07-31 04:15:36 +00:00
committed by GitHub
parent bebf918044
commit 0f17394564
4 changed files with 92 additions and 20 deletions
@@ -159,6 +159,7 @@ def test_pooling_runner_rejects_unsupported_selected_task() -> None:
}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "embed&token_classify"
with (
@@ -166,3 +167,43 @@ def test_pooling_runner_rejects_unsupported_selected_task() -> None:
pytest.raises(ValueError, match="selects 'embed&token_classify'"),
):
PoolingRunner(model, vllm_config)
def test_pooling_runner_supports_encoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"token_classify"}
def test_pooling_runner_rejects_decoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
with pytest.raises(ValueError, match="selects 'token_classify'") as exc_info:
PoolingRunner(model, vllm_config)
assert "Set an explicitly supported task" not in str(exc_info.value)
def test_pooling_runner_filters_decoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "embed"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"embed"}
@@ -29,16 +29,24 @@ def seed_everything():
)
# The float32 is required for this tiny model to pass the test.
@pytest.mark.parametrize("dtype", ["float"])
@pytest.mark.core_model
@torch.inference_mode
def test_bert_models(
def test_bert_model_runner_v2(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
dtype: str,
) -> None:
prompt_batches = [[example_prompts[0]], example_prompts]
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(example_prompts)
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_output_batches = [
vllm_model.token_classify(prompts) for prompts in prompt_batches
]
# Use eager attention on ROCm to avoid HF Transformers flash attention
# accuracy issues: https://github.com/vllm-project/vllm/issues/30167
@@ -59,12 +67,14 @@ def test_bert_models(
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
hf_output_batches = [[hf_outputs[0]], hf_outputs]
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
for hf_outputs, vllm_outputs in zip(hf_output_batches, vllm_output_batches):
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
@pytest.mark.parametrize("model", ["disham993/electrical-ner-ModernBERT-base"])