[Frontend] Add automatic language detection for Whisper transcription (#34342)
Signed-off-by: space_check <roman.vuskov@rwth-aachen.de> Signed-off-by: Roman <45857014+spacecheck@users.noreply.github.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Nicolò Lucchesi
parent
272b535ab3
commit
98b0205c3c
@@ -273,3 +273,30 @@ async def test_audio_with_max_tokens(whisper_client, mary_had_lamb):
|
||||
out_text = out["text"]
|
||||
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
|
||||
assert len(out_tokens) < 450 # ~Whisper max output len
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("fixture_name", "expected_lang", "expected_text"),
|
||||
[
|
||||
("mary_had_lamb", "en", ["Mary had a little lamb"]),
|
||||
("foscolo", "it", ["zacinto", "sacre"]),
|
||||
],
|
||||
ids=["english", "italian"],
|
||||
)
|
||||
async def test_language_auto_detect(
|
||||
whisper_client, fixture_name, expected_lang, expected_text, request
|
||||
):
|
||||
"""Auto-detect language when no language param is provided."""
|
||||
audio_file = request.getfixturevalue(fixture_name)
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=audio_file,
|
||||
response_format="verbose_json",
|
||||
temperature=0.0,
|
||||
)
|
||||
assert transcription.language == expected_lang
|
||||
text_lower = transcription.text.lower()
|
||||
assert any(word.lower() in text_lower for word in expected_text), (
|
||||
f"Expected {expected_lang} text but got: {transcription.text}"
|
||||
)
|
||||
|
||||
@@ -111,6 +111,47 @@ def check_model_available(model: str) -> None:
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
|
||||
def test_parse_language_detection_output():
|
||||
"""Unit test for WhisperForConditionalGeneration.parse_language_detection_output.
|
||||
|
||||
No GPU or model loading required.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.model_executor.models.whisper import (
|
||||
WhisperForConditionalGeneration,
|
||||
)
|
||||
|
||||
cls = WhisperForConditionalGeneration
|
||||
|
||||
def make_tokenizer(return_value: str) -> MagicMock:
|
||||
tok = MagicMock()
|
||||
tok.decode = MagicMock(return_value=return_value)
|
||||
return tok
|
||||
|
||||
# English
|
||||
assert (
|
||||
cls.parse_language_detection_output([50259], make_tokenizer("<|en|>")) == "en"
|
||||
)
|
||||
|
||||
# German
|
||||
assert (
|
||||
cls.parse_language_detection_output([50261], make_tokenizer("<|de|>")) == "de"
|
||||
)
|
||||
|
||||
# Unsupported language code
|
||||
with pytest.raises(AssertionError):
|
||||
cls.parse_language_detection_output([99999], make_tokenizer("<|xx|>"))
|
||||
|
||||
# No special token format
|
||||
with pytest.raises(AssertionError):
|
||||
cls.parse_language_detection_output([1], make_tokenizer("hello"))
|
||||
|
||||
# Empty token_ids
|
||||
with pytest.raises((AssertionError, IndexError)):
|
||||
cls.parse_language_detection_output([], make_tokenizer("anything"))
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu_model
|
||||
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
|
||||
|
||||
Reference in New Issue
Block a user