diff --git a/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py b/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py index af61ebc526..62b1d36784 100644 --- a/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py +++ b/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py @@ -17,7 +17,7 @@ import pytest import soundfile import torch from datasets import Audio, load_dataset -from evaluate import load +from jiwer import wer from transformers.models.whisper.english_normalizer import EnglishTextNormalizer from vllm.benchmarks.datasets.datasets import ASRDataset @@ -202,8 +202,7 @@ def run_evaluation( # Compute WER predictions = [res[2] for res in results] references = [res[3] for res in results] - wer = load("wer") - wer_score = 100 * wer.compute(references=references, predictions=predictions) + wer_score = 100 * wer(references, predictions) print("WER:", wer_score) return wer_score @@ -302,8 +301,7 @@ def run_longform_evaluation( predictions = [res[2] for res in results] references = [res[3] for res in results] - wer = load("wer") - wer_score = 100 * wer.compute(references=references, predictions=predictions) + wer_score = 100 * wer(references, predictions) print("WER:", wer_score) return wer_score