[CI] Consolidate Speech to Text tests (#42274)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-05-11 19:50:17 +00:00
committed by GitHub
parent 56e5810ff1
commit a0dc7a0f36
18 changed files with 115 additions and 68 deletions
@@ -0,0 +1,61 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
@pytest.fixture(scope="module")
def transcription_server_with_force_include_usage():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-num-seqs",
"4",
"--enforce-eager",
"--enable-force-include-usage",
"--gpu-memory-utilization",
"0.2",
]
with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def transcription_client_with_force_include_usage(
transcription_server_with_force_include_usage,
):
async with (
transcription_server_with_force_include_usage.get_async_client() as async_client
):
yield async_client
@pytest.mark.asyncio
async def test_transcription_with_enable_force_include_usage(
transcription_client_with_force_include_usage, winning_call
):
res = (
await transcription_client_with_force_include_usage.audio.transcriptions.create(
model="openai/whisper-large-v3-turbo",
file=winning_call,
language="en",
temperature=0.0,
stream=True,
timeout=30,
)
)
async for chunk in res:
if not len(chunk.choices):
# final usage sent
usage = chunk.usage
assert isinstance(usage, dict)
assert usage["prompt_tokens"] > 0
assert usage["completion_tokens"] > 0
assert usage["total_tokens"] > 0
else:
assert not hasattr(chunk, "usage")
@@ -0,0 +1,271 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ASR inter-chunk spacing: ``asr_inter_chunk_separator`` and transcription
serving (mocked).
Unit tests cover the helper and ``SupportsTranscription.no_space_languages``.
Integration-style tests exercise ``OpenAIServingTranscription`` streaming and
``create_transcription`` without loading a model.
"""
from __future__ import annotations
import json
from collections.abc import AsyncGenerator
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from vllm.config import ModelConfig
from vllm.config.speech_to_text import SpeechToTextConfig
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
RequestResponseMetadata,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.speech_to_text.protocol import TranscriptionRequest
from vllm.entrypoints.openai.speech_to_text.serving import OpenAIServingTranscription
from vllm.entrypoints.openai.speech_to_text.speech_to_text import (
OpenAISpeechToText,
asr_inter_chunk_separator,
)
from vllm.model_executor.models.interfaces import SupportsTranscription
from vllm.outputs import CompletionOutput, RequestOutput
# --- Unit: helper + protocol -------------------------------------------------
def test_default_no_space_languages_includes_zh_and_ja():
assert SupportsTranscription.no_space_languages == {"ja", "zh"}
@pytest.mark.parametrize(
("language", "expected_sep"),
[
("en", " "),
("EN", " "),
("zh", ""),
("ZH", ""),
("ja", ""),
(None, " "),
],
)
def test_asr_inter_chunk_separator_matches_protocol(language, expected_sep):
sep = asr_inter_chunk_separator(language, SupportsTranscription.no_space_languages)
assert sep == expected_sep
def test_joined_chunks_english_has_space_between():
sep = asr_inter_chunk_separator("en", SupportsTranscription.no_space_languages)
assert sep.join(["hello", "world"]) == "hello world"
def test_joined_chunks_chinese_has_no_space_between():
sep = asr_inter_chunk_separator("zh", SupportsTranscription.no_space_languages)
assert sep.join(["你好", "世界"]) == "你好世界"
# --- Integration: serving (no model) -----------------------------------------
class _StubTranscriptionModel:
"""Minimal stand-in for a SupportsTranscription implementation (no torch)."""
no_space_languages: set[str] = {"ja", "zh"}
supports_segment_timestamp = False
@classmethod
def get_speech_to_text_config(
cls, model_config: ModelConfig, task_type: str
) -> SpeechToTextConfig:
return SpeechToTextConfig(
sample_rate=16000.0,
max_audio_clip_s=5.0,
)
@classmethod
def post_process_output(cls, text: str) -> str:
return text
def _request_output(text: str) -> RequestOutput:
return RequestOutput(
request_id="rid",
prompt=None,
prompt_token_ids=None,
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text=text,
token_ids=(1, 2, 3),
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
],
finished=True,
)
def _sse_delta_contents(sse_body: str) -> list[str]:
"""Extract ``choices[0].delta.content`` from each ``data:`` line (streaming API)."""
contents: list[str] = []
for line in sse_body.splitlines():
if not line.startswith("data: "):
continue
payload = line.removeprefix("data: ").strip()
if payload == "[DONE]":
continue
obj = json.loads(payload)
for choice in obj.get("choices") or []:
delta = choice.get("delta") or {}
if "content" in delta:
contents.append(delta["content"])
return contents
@pytest.mark.asyncio
async def test_transcription_stream_generator_english_inserts_space_between_chunks():
"""Online streaming: first output per audio chunk is prefixed with *separator*."""
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("hello")
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("world")
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
serving.enable_force_include_usage = False
serving.model_cls = _StubTranscriptionModel
serving.task_type = "transcribe"
request = SimpleNamespace(
model="stub-model",
stream_include_usage=False,
stream_continuous_usage_stats=False,
)
sep = asr_inter_chunk_separator("en", _StubTranscriptionModel.no_space_languages)
assert sep == " "
out_lines: list[str] = []
agen = OpenAIServingTranscription.transcription_stream_generator(
serving,
request=request,
result_generator=[gen_hello(), gen_world()],
request_id="test-req",
request_metadata=RequestResponseMetadata(request_id="test-req"),
audio_duration_s=1.0,
separator=sep,
)
async for line in agen:
out_lines.append(line)
sse = "".join(out_lines)
combined = "".join(_sse_delta_contents(sse))
assert combined.strip() == "hello world"
@pytest.mark.asyncio
async def test_transcription_stream_generator_chinese_no_space_between_chunks():
async def gen_a() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("你好")
async def gen_b() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("世界")
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
serving.enable_force_include_usage = False
serving.model_cls = _StubTranscriptionModel
serving.task_type = "transcribe"
request = SimpleNamespace(
model="stub-model",
stream_include_usage=False,
stream_continuous_usage_stats=False,
)
sep = asr_inter_chunk_separator("zh", _StubTranscriptionModel.no_space_languages)
assert sep == ""
out_lines: list[str] = []
agen = OpenAIServingTranscription.transcription_stream_generator(
serving,
request=request,
result_generator=[gen_a(), gen_b()],
request_id="test-req-zh",
request_metadata=RequestResponseMetadata(request_id="test-req-zh"),
audio_duration_s=1.0,
separator=sep,
)
async for line in agen:
out_lines.append(line)
combined = "".join(_sse_delta_contents("".join(out_lines)))
assert combined == "你好世界"
@pytest.mark.asyncio
async def test_create_transcription_non_streaming_joins_chunks_by_language():
"""``create_transcription`` uses the same separator logic as the helper."""
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("hello")
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("world")
engine_client = MagicMock()
engine_client.model_config = MagicMock()
engine_client.model_config.get_diff_sampling_param.return_value = {
"max_tokens": 256,
"temperature": 0.0,
}
engine_client.model_config.max_model_len = 8192
engine_client.errored = False
engine_client.generate.side_effect = [gen_hello(), gen_world()]
models = MagicMock(spec=OpenAIServingModels)
models.lora_requests = {}
models.is_base_model.return_value = True
preprocess_mock = AsyncMock(return_value=([MagicMock(), MagicMock()], 1.0))
with (
patch(
"vllm.model_executor.model_loader.get_model_cls",
return_value=_StubTranscriptionModel,
),
patch.object(OpenAISpeechToText, "_preprocess_speech_to_text", preprocess_mock),
):
serving = OpenAIServingTranscription(engine_client, models, request_logger=None)
req_en = TranscriptionRequest.model_construct(
file=MagicMock(),
model="stub-model",
language="en",
stream=False,
response_format="json",
)
out_en = await serving.create_transcription(
b"\x00\x00", req_en, raw_request=None
)
assert not isinstance(out_en, ErrorResponse)
assert out_en.text == "hello world"
async def gen_nihao() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("你好")
async def gen_shijie() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("世界")
engine_client.generate.side_effect = [gen_nihao(), gen_shijie()]
req_zh = TranscriptionRequest.model_construct(
file=MagicMock(),
model="stub-model",
language="zh",
stream=False,
response_format="json",
)
out_zh = await serving.create_transcription(
b"\x00\x00", req_zh, raw_request=None
)
assert not isinstance(out_zh, ErrorResponse)
assert out_zh.text == "你好世界"
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# imports for structured outputs tests
import json
import pytest
from tests.entrypoints.speech_to_text.conftest import add_attention_backend
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
MISTRAL_FORMAT_ARGS = [
"--tokenizer_mode",
"mistral",
"--config_format",
"mistral",
"--load_format",
"mistral",
]
async def transcribe_and_check(
client,
model_name: str,
file,
*,
language: str,
expected_text: str,
expected_seconds: int | None = None,
case_sensitive: bool = False,
):
"""Run a transcription request and assert the output contains
*expected_text* and optionally that usage reports *expected_seconds*.
Provides detailed failure messages with the actual transcription output.
"""
transcription = await client.audio.transcriptions.create(
model=model_name,
file=file,
language=language,
response_format="text",
temperature=0.0,
)
out = json.loads(transcription)
out_text = out["text"]
out_usage = out["usage"]
if case_sensitive:
assert expected_text in out_text, (
f"Expected {expected_text!r} in transcription output, got: {out_text!r}"
)
else:
assert expected_text.lower() in out_text.lower(), (
f"Expected {expected_text!r} (case-insensitive) in transcription "
f"output, got: {out_text!r}"
)
if expected_seconds is not None:
assert out_usage["seconds"] == expected_seconds, (
f"Expected {expected_seconds}s of audio, "
f"got {out_usage['seconds']}s. Full usage: {out_usage!r}"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name", ["mistralai/Voxtral-Mini-3B-2507", "Qwen/Qwen3-ASR-0.6B"]
)
async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
add_attention_backend(server_args, rocm_aiter_fa_attention)
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
with RemoteOpenAIServer(
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
) as remote_server:
client = remote_server.get_async_client()
await transcribe_and_check(
client,
model_name,
mary_had_lamb,
language="en",
expected_text="Mary had a little lamb",
expected_seconds=16,
)
@pytest.mark.asyncio
async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
"""Ensure STT (transcribe) requests can pass LoRA through to generate."""
# ROCm SPECIFIC CONFIGURATION:
# To ensure the test passes on ROCm, we modify the max model length to 512.
# We DO NOT apply this to other platforms to maintain strict upstream parity.
from vllm.platforms import current_platform
model_name = "ibm-granite/granite-speech-3.3-2b"
lora_model_name = "speech"
server_args = [
"--enforce-eager",
"--enable-lora",
"--max-lora-rank",
"64",
"--lora-modules",
f"{lora_model_name}={model_name}",
"--max-model-len",
"512" if current_platform.is_rocm() else "2048",
"--max-num-seqs",
"1",
]
add_attention_backend(server_args, rocm_aiter_fa_attention)
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
with RemoteOpenAIServer(
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
) as remote_server:
client = remote_server.get_async_client()
await transcribe_and_check(
client,
lora_model_name,
mary_had_lamb,
language="en",
expected_text="mary had a little lamb",
expected_seconds=16,
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name", ["google/gemma-3n-E2B-it", "Qwen/Qwen3-ASR-0.6B"]
)
async def test_basic_audio_foscolo(foscolo, rocm_aiter_fa_attention, model_name):
# Gemma accuracy on some of the audio samples we use is particularly bad,
# hence we use a different one here. WER is evaluated separately.
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
add_attention_backend(server_args, rocm_aiter_fa_attention)
with RemoteOpenAIServer(
model_name,
server_args,
max_wait_seconds=480,
env_dict=ROCM_ENV_OVERRIDES,
) as remote_server:
client = remote_server.get_async_client()
await transcribe_and_check(
client,
model_name,
foscolo,
language="it",
expected_text="ove il mio corpo fanciulletto",
)
@@ -0,0 +1,437 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# imports for structured outputs tests
import asyncio
import io
import json
import numpy as np
import openai
import pytest
import pytest_asyncio
import soundfile as sf
from tests.utils import RemoteOpenAIServer
from vllm.multimodal.media.audio import load_audio
from vllm.platforms import current_platform
MODEL_NAME = "openai/whisper-large-v3-turbo"
# Disable prefix caching on ROCm to reduce non-determinism in
# streaming-vs-non-streaming comparisons.
_ROCM_ARGS = ["--no-enable-prefix-caching"] if current_platform.is_rocm() else []
def _get_attention_backend_params() -> list[str | None]:
"""Return attention backends to parametrize the server fixture with.
On ROCm, we test multiple backends explicitly:
- None: default auto-selection (ROCM_ATTN for decoder self-attention,
falls back to ROCM_AITER_UNIFIED_ATTN or TRITON_ATTN for
cross-attention since ROCM_ATTN doesn't support ENCODER_DECODER)
- TRITON_ATTN: always available on ROCm
- ROCM_AITER_UNIFIED_ATTN: only on gfx942/gfx950
On non-ROCm platforms, we just run with the default backend.
"""
try:
from vllm.platforms import current_platform
if current_platform.is_rocm():
backends: list[str | None] = [None, "TRITON_ATTN"]
from vllm.platforms.rocm import _ON_MI3XX
if _ON_MI3XX:
backends.append("ROCM_AITER_UNIFIED_ATTN")
return backends
except Exception:
pass
return [None]
# Aiter backends need VLLM_ROCM_USE_AITER=1 (and MHA=1 for ROCM_AITER_FA)
# to be enabled in the server subprocess.
_AITER_ENV = {
"VLLM_ROCM_USE_AITER": "1",
"VLLM_ROCM_USE_AITER_MHA": "1",
}
_ATTN_BACKENDS = _get_attention_backend_params()
_ATTN_IDS = [b or "default" for b in _ATTN_BACKENDS]
@pytest.fixture(scope="module", params=_ATTN_BACKENDS, ids=_ATTN_IDS)
def server(request):
args = [*_ROCM_ARGS]
env_dict = None
if request.param is not None:
args += ["--attention-backend", request.param]
if "AITER" in request.param:
env_dict = _AITER_ENV
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def whisper_client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
async def test_basic_audio(whisper_client, mary_had_lamb):
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
response_format="text",
temperature=0.0,
)
out = json.loads(transcription)
out_text = out["text"]
out_usage = out["usage"]
assert "Mary had a little lamb," in out_text
assert out_usage["seconds"] == 16, out_usage["seconds"]
@pytest.mark.asyncio
async def test_basic_audio_batched(mary_had_lamb, winning_call, whisper_client):
transcription = whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
response_format="text",
temperature=0.0,
)
transcription2 = whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=winning_call,
language="en",
response_format="text",
temperature=0.0,
)
# Await both transcriptions by scheduling coroutines together
transcription, transcription2 = await asyncio.gather(transcription, transcription2)
out = json.loads(transcription)
out_text = out["text"]
assert "Mary had a little lamb," in out_text
out2 = json.loads(transcription2)
out_text2 = out2["text"]
assert "Edgar Martinez" in out_text2
@pytest.mark.asyncio
async def test_bad_requests(mary_had_lamb, whisper_client):
# invalid language
with pytest.raises(openai.BadRequestError):
await whisper_client.audio.transcriptions.create(
model=MODEL_NAME, file=mary_had_lamb, language="hh", temperature=0.0
)
@pytest.mark.asyncio
async def test_long_audio_request(mary_had_lamb, whisper_client):
mary_had_lamb.seek(0)
audio, sr = load_audio(mary_had_lamb)
# Add small silence after each audio for repeatability in the split process
audio = np.pad(audio, (0, 1600))
repeated_audio = np.tile(audio, 10)
# Repeated audio to buffer
buffer = io.BytesIO()
sf.write(buffer, repeated_audio, sr, format="WAV")
buffer.seek(0)
transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=buffer,
language="en",
response_format="text",
temperature=0.0,
)
out = json.loads(transcription)
out_text = out["text"]
out_usage = out["usage"]
counts = out_text.count("Mary had a little lamb")
assert counts == 10, counts
assert out_usage["seconds"] == 161, out_usage["seconds"]
@pytest.mark.asyncio
async def test_invalid_audio_file(whisper_client):
"""Corrupted audio should surface as HTTP 400."""
invalid_audio = io.BytesIO(b"not a valid audio file")
invalid_audio.name = "invalid.wav"
with pytest.raises(openai.BadRequestError) as exc_info:
await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=invalid_audio,
language="en",
)
assert exc_info.value.status_code == 400
assert "Invalid or unsupported audio file" in exc_info.value.message
@pytest.mark.asyncio
async def test_completion_endpoints(whisper_client):
# text to text model
with pytest.raises(openai.NotFoundError):
await whisper_client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "system", "content": "You are a helpful assistant."}],
)
with pytest.raises(openai.NotFoundError):
await whisper_client.completions.create(model=MODEL_NAME, prompt="Hello")
@pytest.mark.asyncio
async def test_streaming_response(winning_call, whisper_client):
transcription = ""
res_no_stream = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=winning_call,
response_format="json",
language="en",
temperature=0.0,
)
res = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=winning_call,
language="en",
temperature=0.0,
stream=True,
timeout=30,
)
# Reconstruct from chunks and validate
async for chunk in res:
text = chunk.choices[0]["delta"]["content"]
transcription += text
assert transcription == res_no_stream.text
@pytest.mark.asyncio
async def test_stream_options(winning_call, whisper_client):
res = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=winning_call,
language="en",
temperature=0.0,
stream=True,
extra_body=dict(stream_include_usage=True, stream_continuous_usage_stats=True),
timeout=30,
)
final = False
continuous = True
async for chunk in res:
if not len(chunk.choices):
# final usage sent
final = True
else:
continuous = continuous and hasattr(chunk, "usage")
assert final and continuous
@pytest.mark.asyncio
async def test_sampling_params(mary_had_lamb, whisper_client):
"""
Compare sampling with params and greedy sampling to assert results
are different when extreme sampling parameters values are picked.
"""
transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
temperature=0.8,
extra_body=dict(
seed=42,
repetition_penalty=1.9,
top_k=12,
top_p=0.4,
min_p=0.5,
frequency_penalty=1.8,
presence_penalty=2.0,
),
)
greedy_transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
temperature=0.0,
extra_body=dict(seed=42),
)
assert greedy_transcription.text != transcription.text
@pytest.mark.asyncio
async def test_audio_prompt(mary_had_lamb, whisper_client):
prompt = "This is a speech, recorded in a phonograph."
# Prompts should not omit the part of original prompt while transcribing.
prefix = "The first words I spoke in the original phonograph"
transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
response_format="text",
temperature=0.0,
)
out = json.loads(transcription)["text"]
assert prefix in out
transcription_wprompt = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
response_format="text",
prompt=prompt,
temperature=0.0,
)
out_prompt = json.loads(transcription_wprompt)["text"]
assert prefix in out_prompt
@pytest.mark.asyncio
async def test_audio_with_timestamp(mary_had_lamb, whisper_client):
transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
response_format="verbose_json",
temperature=0.0,
)
assert transcription.segments is not None
assert len(transcription.segments) > 0
assert transcription.segments[0].avg_logprob is not None
assert transcription.segments[0].compression_ratio is not None
@pytest.mark.asyncio
async def test_audio_with_max_tokens(whisper_client, mary_had_lamb):
transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
response_format="text",
temperature=0.0,
extra_body={"max_completion_tokens": 1},
)
out = json.loads(transcription)
out_text = out["text"]
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(MODEL_NAME)
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
assert len(out_tokens) == 1
# max_completion_tokens > max_model_len
transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
response_format="text",
temperature=0.0,
extra_body={"max_completion_tokens": int(1e6)},
)
out = json.loads(transcription)
out_text = out["text"]
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
assert len(out_tokens) < 450 # ~Whisper max output len
@pytest.mark.asyncio
@pytest.mark.parametrize(
("fixture_name", "expected_lang", "expected_text"),
[
("mary_had_lamb", "en", ["Mary had a little lamb"]),
("foscolo", "it", ["zacinto", "sacre"]),
],
ids=["english", "italian"],
)
async def test_language_auto_detect(
whisper_client, fixture_name, expected_lang, expected_text, request
):
"""Auto-detect language when no language param is provided."""
audio_file = request.getfixturevalue(fixture_name)
transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=audio_file,
response_format="verbose_json",
temperature=0.0,
)
assert transcription.language == expected_lang
text_lower = transcription.text.lower()
assert any(word.lower() in text_lower for word in expected_text), (
f"Expected {expected_lang} text but got: {transcription.text}"
)
@pytest.mark.asyncio
async def test_whisper_beam_search_single_beam(mary_had_lamb, whisper_client):
"""Test beam search with encoder-decoder model (Whisper) on transcriptions with
one beam aligns with greedy decoding.
"""
beam_transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
response_format="text",
temperature=0.0,
extra_body=dict(
use_beam_search=True,
n=1,
),
)
greedy_transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
response_format="text",
temperature=0.0,
)
greedy_res = json.loads(greedy_transcription)["text"]
beam_res = json.loads(beam_transcription)["text"]
assert greedy_res == beam_res
@pytest.mark.asyncio
async def test_whisper_beam_search_multibeam(mary_had_lamb, whisper_client):
"""Test n>1 for beam search returns one transcription (best beam)."""
transcription = await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=mary_had_lamb,
language="en",
response_format="text",
temperature=0.0,
extra_body=dict(
use_beam_search=True,
n=2,
),
)
result = json.loads(transcription)
text = result["text"]
assert text is not None
assert len(text) > 0
assert "mary had a little lamb" in text.lower()
@pytest.mark.asyncio
async def test_stream_with_beams_raises(winning_call, whisper_client):
"""Test that stream=True + beam search raises bad request for now."""
with pytest.raises(openai.BadRequestError):
await whisper_client.audio.transcriptions.create(
model=MODEL_NAME,
file=winning_call,
language="en",
stream=True,
extra_body=dict(
use_beam_search=True,
n=2,
),
)