[Model Runner V2] Enable encoder token classification (#50293)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -29,16 +29,24 @@ def seed_everything():
|
||||
)
|
||||
# The float32 is required for this tiny model to pass the test.
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@pytest.mark.core_model
|
||||
@torch.inference_mode
|
||||
def test_bert_models(
|
||||
def test_bert_model_runner_v2(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
prompt_batches = [[example_prompts[0]], example_prompts]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_output_batches = [
|
||||
vllm_model.token_classify(prompts) for prompts in prompt_batches
|
||||
]
|
||||
|
||||
# Use eager attention on ROCm to avoid HF Transformers flash attention
|
||||
# accuracy issues: https://github.com/vllm-project/vllm/issues/30167
|
||||
@@ -59,12 +67,14 @@ def test_bert_models(
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
hf_output_batches = [[hf_outputs[0]], hf_outputs]
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
for hf_outputs, vllm_outputs in zip(hf_output_batches, vllm_output_batches):
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["disham993/electrical-ner-ModernBERT-base"])
|
||||
|
||||
Reference in New Issue
Block a user