[CI] Consolidate Speech to Text tests (#42274)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def transcription_server_with_force_include_usage():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--enforce-eager",
|
||||
"--enable-force-include-usage",
|
||||
"--gpu-memory-utilization",
|
||||
"0.2",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def transcription_client_with_force_include_usage(
|
||||
transcription_server_with_force_include_usage,
|
||||
):
|
||||
async with (
|
||||
transcription_server_with_force_include_usage.get_async_client() as async_client
|
||||
):
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_with_enable_force_include_usage(
|
||||
transcription_client_with_force_include_usage, winning_call
|
||||
):
|
||||
res = (
|
||||
await transcription_client_with_force_include_usage.audio.transcriptions.create(
|
||||
model="openai/whisper-large-v3-turbo",
|
||||
file=winning_call,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
timeout=30,
|
||||
)
|
||||
)
|
||||
|
||||
async for chunk in res:
|
||||
if not len(chunk.choices):
|
||||
# final usage sent
|
||||
usage = chunk.usage
|
||||
assert isinstance(usage, dict)
|
||||
assert usage["prompt_tokens"] > 0
|
||||
assert usage["completion_tokens"] > 0
|
||||
assert usage["total_tokens"] > 0
|
||||
else:
|
||||
assert not hasattr(chunk, "usage")
|
||||
+271
@@ -0,0 +1,271 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ASR inter-chunk spacing: ``asr_inter_chunk_separator`` and transcription
|
||||
serving (mocked).
|
||||
|
||||
Unit tests cover the helper and ``SupportsTranscription.no_space_languages``.
|
||||
Integration-style tests exercise ``OpenAIServingTranscription`` streaming and
|
||||
``create_transcription`` without loading a model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.speech_to_text import SpeechToTextConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.speech_to_text.protocol import TranscriptionRequest
|
||||
from vllm.entrypoints.openai.speech_to_text.serving import OpenAIServingTranscription
|
||||
from vllm.entrypoints.openai.speech_to_text.speech_to_text import (
|
||||
OpenAISpeechToText,
|
||||
asr_inter_chunk_separator,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import SupportsTranscription
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
|
||||
# --- Unit: helper + protocol -------------------------------------------------
|
||||
|
||||
|
||||
def test_default_no_space_languages_includes_zh_and_ja():
|
||||
assert SupportsTranscription.no_space_languages == {"ja", "zh"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("language", "expected_sep"),
|
||||
[
|
||||
("en", " "),
|
||||
("EN", " "),
|
||||
("zh", ""),
|
||||
("ZH", ""),
|
||||
("ja", ""),
|
||||
(None, " "),
|
||||
],
|
||||
)
|
||||
def test_asr_inter_chunk_separator_matches_protocol(language, expected_sep):
|
||||
sep = asr_inter_chunk_separator(language, SupportsTranscription.no_space_languages)
|
||||
assert sep == expected_sep
|
||||
|
||||
|
||||
def test_joined_chunks_english_has_space_between():
|
||||
sep = asr_inter_chunk_separator("en", SupportsTranscription.no_space_languages)
|
||||
assert sep.join(["hello", "world"]) == "hello world"
|
||||
|
||||
|
||||
def test_joined_chunks_chinese_has_no_space_between():
|
||||
sep = asr_inter_chunk_separator("zh", SupportsTranscription.no_space_languages)
|
||||
assert sep.join(["你好", "世界"]) == "你好世界"
|
||||
|
||||
|
||||
# --- Integration: serving (no model) -----------------------------------------
|
||||
|
||||
|
||||
class _StubTranscriptionModel:
|
||||
"""Minimal stand-in for a SupportsTranscription implementation (no torch)."""
|
||||
|
||||
no_space_languages: set[str] = {"ja", "zh"}
|
||||
supports_segment_timestamp = False
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls, model_config: ModelConfig, task_type: str
|
||||
) -> SpeechToTextConfig:
|
||||
return SpeechToTextConfig(
|
||||
sample_rate=16000.0,
|
||||
max_audio_clip_s=5.0,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def post_process_output(cls, text: str) -> str:
|
||||
return text
|
||||
|
||||
|
||||
def _request_output(text: str) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id="rid",
|
||||
prompt=None,
|
||||
prompt_token_ids=None,
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
token_ids=(1, 2, 3),
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
|
||||
def _sse_delta_contents(sse_body: str) -> list[str]:
|
||||
"""Extract ``choices[0].delta.content`` from each ``data:`` line (streaming API)."""
|
||||
contents: list[str] = []
|
||||
for line in sse_body.splitlines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
payload = line.removeprefix("data: ").strip()
|
||||
if payload == "[DONE]":
|
||||
continue
|
||||
obj = json.loads(payload)
|
||||
for choice in obj.get("choices") or []:
|
||||
delta = choice.get("delta") or {}
|
||||
if "content" in delta:
|
||||
contents.append(delta["content"])
|
||||
return contents
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_stream_generator_english_inserts_space_between_chunks():
|
||||
"""Online streaming: first output per audio chunk is prefixed with *separator*."""
|
||||
|
||||
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("hello")
|
||||
|
||||
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("world")
|
||||
|
||||
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
|
||||
serving.enable_force_include_usage = False
|
||||
serving.model_cls = _StubTranscriptionModel
|
||||
serving.task_type = "transcribe"
|
||||
request = SimpleNamespace(
|
||||
model="stub-model",
|
||||
stream_include_usage=False,
|
||||
stream_continuous_usage_stats=False,
|
||||
)
|
||||
sep = asr_inter_chunk_separator("en", _StubTranscriptionModel.no_space_languages)
|
||||
assert sep == " "
|
||||
|
||||
out_lines: list[str] = []
|
||||
agen = OpenAIServingTranscription.transcription_stream_generator(
|
||||
serving,
|
||||
request=request,
|
||||
result_generator=[gen_hello(), gen_world()],
|
||||
request_id="test-req",
|
||||
request_metadata=RequestResponseMetadata(request_id="test-req"),
|
||||
audio_duration_s=1.0,
|
||||
separator=sep,
|
||||
)
|
||||
async for line in agen:
|
||||
out_lines.append(line)
|
||||
sse = "".join(out_lines)
|
||||
combined = "".join(_sse_delta_contents(sse))
|
||||
assert combined.strip() == "hello world"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_stream_generator_chinese_no_space_between_chunks():
|
||||
async def gen_a() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("你好")
|
||||
|
||||
async def gen_b() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("世界")
|
||||
|
||||
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
|
||||
serving.enable_force_include_usage = False
|
||||
serving.model_cls = _StubTranscriptionModel
|
||||
serving.task_type = "transcribe"
|
||||
request = SimpleNamespace(
|
||||
model="stub-model",
|
||||
stream_include_usage=False,
|
||||
stream_continuous_usage_stats=False,
|
||||
)
|
||||
sep = asr_inter_chunk_separator("zh", _StubTranscriptionModel.no_space_languages)
|
||||
assert sep == ""
|
||||
|
||||
out_lines: list[str] = []
|
||||
agen = OpenAIServingTranscription.transcription_stream_generator(
|
||||
serving,
|
||||
request=request,
|
||||
result_generator=[gen_a(), gen_b()],
|
||||
request_id="test-req-zh",
|
||||
request_metadata=RequestResponseMetadata(request_id="test-req-zh"),
|
||||
audio_duration_s=1.0,
|
||||
separator=sep,
|
||||
)
|
||||
async for line in agen:
|
||||
out_lines.append(line)
|
||||
combined = "".join(_sse_delta_contents("".join(out_lines)))
|
||||
assert combined == "你好世界"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_transcription_non_streaming_joins_chunks_by_language():
|
||||
"""``create_transcription`` uses the same separator logic as the helper."""
|
||||
|
||||
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("hello")
|
||||
|
||||
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("world")
|
||||
|
||||
engine_client = MagicMock()
|
||||
engine_client.model_config = MagicMock()
|
||||
engine_client.model_config.get_diff_sampling_param.return_value = {
|
||||
"max_tokens": 256,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
engine_client.model_config.max_model_len = 8192
|
||||
engine_client.errored = False
|
||||
engine_client.generate.side_effect = [gen_hello(), gen_world()]
|
||||
|
||||
models = MagicMock(spec=OpenAIServingModels)
|
||||
models.lora_requests = {}
|
||||
models.is_base_model.return_value = True
|
||||
|
||||
preprocess_mock = AsyncMock(return_value=([MagicMock(), MagicMock()], 1.0))
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.model_loader.get_model_cls",
|
||||
return_value=_StubTranscriptionModel,
|
||||
),
|
||||
patch.object(OpenAISpeechToText, "_preprocess_speech_to_text", preprocess_mock),
|
||||
):
|
||||
serving = OpenAIServingTranscription(engine_client, models, request_logger=None)
|
||||
|
||||
req_en = TranscriptionRequest.model_construct(
|
||||
file=MagicMock(),
|
||||
model="stub-model",
|
||||
language="en",
|
||||
stream=False,
|
||||
response_format="json",
|
||||
)
|
||||
out_en = await serving.create_transcription(
|
||||
b"\x00\x00", req_en, raw_request=None
|
||||
)
|
||||
assert not isinstance(out_en, ErrorResponse)
|
||||
assert out_en.text == "hello world"
|
||||
|
||||
async def gen_nihao() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("你好")
|
||||
|
||||
async def gen_shijie() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("世界")
|
||||
|
||||
engine_client.generate.side_effect = [gen_nihao(), gen_shijie()]
|
||||
|
||||
req_zh = TranscriptionRequest.model_construct(
|
||||
file=MagicMock(),
|
||||
model="stub-model",
|
||||
language="zh",
|
||||
stream=False,
|
||||
response_format="json",
|
||||
)
|
||||
out_zh = await serving.create_transcription(
|
||||
b"\x00\x00", req_zh, raw_request=None
|
||||
)
|
||||
assert not isinstance(out_zh, ErrorResponse)
|
||||
assert out_zh.text == "你好世界"
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# imports for structured outputs tests
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.speech_to_text.conftest import add_attention_backend
|
||||
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
|
||||
MISTRAL_FORMAT_ARGS = [
|
||||
"--tokenizer_mode",
|
||||
"mistral",
|
||||
"--config_format",
|
||||
"mistral",
|
||||
"--load_format",
|
||||
"mistral",
|
||||
]
|
||||
|
||||
|
||||
async def transcribe_and_check(
|
||||
client,
|
||||
model_name: str,
|
||||
file,
|
||||
*,
|
||||
language: str,
|
||||
expected_text: str,
|
||||
expected_seconds: int | None = None,
|
||||
case_sensitive: bool = False,
|
||||
):
|
||||
"""Run a transcription request and assert the output contains
|
||||
*expected_text* and optionally that usage reports *expected_seconds*.
|
||||
|
||||
Provides detailed failure messages with the actual transcription output.
|
||||
"""
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
model=model_name,
|
||||
file=file,
|
||||
language=language,
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
out_usage = out["usage"]
|
||||
|
||||
if case_sensitive:
|
||||
assert expected_text in out_text, (
|
||||
f"Expected {expected_text!r} in transcription output, got: {out_text!r}"
|
||||
)
|
||||
else:
|
||||
assert expected_text.lower() in out_text.lower(), (
|
||||
f"Expected {expected_text!r} (case-insensitive) in transcription "
|
||||
f"output, got: {out_text!r}"
|
||||
)
|
||||
|
||||
if expected_seconds is not None:
|
||||
assert out_usage["seconds"] == expected_seconds, (
|
||||
f"Expected {expected_seconds}s of audio, "
|
||||
f"got {out_usage['seconds']}s. Full usage: {out_usage!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name", ["mistralai/Voxtral-Mini-3B-2507", "Qwen/Qwen3-ASR-0.6B"]
|
||||
)
|
||||
async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):
|
||||
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
|
||||
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
await transcribe_and_check(
|
||||
client,
|
||||
model_name,
|
||||
mary_had_lamb,
|
||||
language="en",
|
||||
expected_text="Mary had a little lamb",
|
||||
expected_seconds=16,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
|
||||
"""Ensure STT (transcribe) requests can pass LoRA through to generate."""
|
||||
# ROCm SPECIFIC CONFIGURATION:
|
||||
# To ensure the test passes on ROCm, we modify the max model length to 512.
|
||||
# We DO NOT apply this to other platforms to maintain strict upstream parity.
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
model_name = "ibm-granite/granite-speech-3.3-2b"
|
||||
lora_model_name = "speech"
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
"--enable-lora",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--lora-modules",
|
||||
f"{lora_model_name}={model_name}",
|
||||
"--max-model-len",
|
||||
"512" if current_platform.is_rocm() else "2048",
|
||||
"--max-num-seqs",
|
||||
"1",
|
||||
]
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
await transcribe_and_check(
|
||||
client,
|
||||
lora_model_name,
|
||||
mary_had_lamb,
|
||||
language="en",
|
||||
expected_text="mary had a little lamb",
|
||||
expected_seconds=16,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name", ["google/gemma-3n-E2B-it", "Qwen/Qwen3-ASR-0.6B"]
|
||||
)
|
||||
async def test_basic_audio_foscolo(foscolo, rocm_aiter_fa_attention, model_name):
|
||||
# Gemma accuracy on some of the audio samples we use is particularly bad,
|
||||
# hence we use a different one here. WER is evaluated separately.
|
||||
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
max_wait_seconds=480,
|
||||
env_dict=ROCM_ENV_OVERRIDES,
|
||||
) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
await transcribe_and_check(
|
||||
client,
|
||||
model_name,
|
||||
foscolo,
|
||||
language="it",
|
||||
expected_text="ove il mio corpo fanciulletto",
|
||||
)
|
||||
+437
@@ -0,0 +1,437 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# imports for structured outputs tests
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import soundfile as sf
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "openai/whisper-large-v3-turbo"
|
||||
|
||||
# Disable prefix caching on ROCm to reduce non-determinism in
|
||||
# streaming-vs-non-streaming comparisons.
|
||||
_ROCM_ARGS = ["--no-enable-prefix-caching"] if current_platform.is_rocm() else []
|
||||
|
||||
|
||||
def _get_attention_backend_params() -> list[str | None]:
|
||||
"""Return attention backends to parametrize the server fixture with.
|
||||
|
||||
On ROCm, we test multiple backends explicitly:
|
||||
- None: default auto-selection (ROCM_ATTN for decoder self-attention,
|
||||
falls back to ROCM_AITER_UNIFIED_ATTN or TRITON_ATTN for
|
||||
cross-attention since ROCM_ATTN doesn't support ENCODER_DECODER)
|
||||
- TRITON_ATTN: always available on ROCm
|
||||
- ROCM_AITER_UNIFIED_ATTN: only on gfx942/gfx950
|
||||
|
||||
On non-ROCm platforms, we just run with the default backend.
|
||||
"""
|
||||
try:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
backends: list[str | None] = [None, "TRITON_ATTN"]
|
||||
from vllm.platforms.rocm import _ON_MI3XX
|
||||
|
||||
if _ON_MI3XX:
|
||||
backends.append("ROCM_AITER_UNIFIED_ATTN")
|
||||
return backends
|
||||
except Exception:
|
||||
pass
|
||||
return [None]
|
||||
|
||||
|
||||
# Aiter backends need VLLM_ROCM_USE_AITER=1 (and MHA=1 for ROCM_AITER_FA)
|
||||
# to be enabled in the server subprocess.
|
||||
_AITER_ENV = {
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
"VLLM_ROCM_USE_AITER_MHA": "1",
|
||||
}
|
||||
|
||||
_ATTN_BACKENDS = _get_attention_backend_params()
|
||||
_ATTN_IDS = [b or "default" for b in _ATTN_BACKENDS]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=_ATTN_BACKENDS, ids=_ATTN_IDS)
|
||||
def server(request):
|
||||
args = [*_ROCM_ARGS]
|
||||
env_dict = None
|
||||
if request.param is not None:
|
||||
args += ["--attention-backend", request.param]
|
||||
if "AITER" in request.param:
|
||||
env_dict = _AITER_ENV
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def whisper_client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_audio(whisper_client, mary_had_lamb):
|
||||
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
out_usage = out["usage"]
|
||||
assert "Mary had a little lamb," in out_text
|
||||
assert out_usage["seconds"] == 16, out_usage["seconds"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_audio_batched(mary_had_lamb, winning_call, whisper_client):
|
||||
transcription = whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
transcription2 = whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
# Await both transcriptions by scheduling coroutines together
|
||||
transcription, transcription2 = await asyncio.gather(transcription, transcription2)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
assert "Mary had a little lamb," in out_text
|
||||
out2 = json.loads(transcription2)
|
||||
out_text2 = out2["text"]
|
||||
assert "Edgar Martinez" in out_text2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bad_requests(mary_had_lamb, whisper_client):
|
||||
# invalid language
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME, file=mary_had_lamb, language="hh", temperature=0.0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_audio_request(mary_had_lamb, whisper_client):
|
||||
mary_had_lamb.seek(0)
|
||||
audio, sr = load_audio(mary_had_lamb)
|
||||
# Add small silence after each audio for repeatability in the split process
|
||||
audio = np.pad(audio, (0, 1600))
|
||||
repeated_audio = np.tile(audio, 10)
|
||||
# Repeated audio to buffer
|
||||
buffer = io.BytesIO()
|
||||
sf.write(buffer, repeated_audio, sr, format="WAV")
|
||||
buffer.seek(0)
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=buffer,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
out_usage = out["usage"]
|
||||
counts = out_text.count("Mary had a little lamb")
|
||||
assert counts == 10, counts
|
||||
assert out_usage["seconds"] == 161, out_usage["seconds"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_audio_file(whisper_client):
|
||||
"""Corrupted audio should surface as HTTP 400."""
|
||||
invalid_audio = io.BytesIO(b"not a valid audio file")
|
||||
invalid_audio.name = "invalid.wav"
|
||||
|
||||
with pytest.raises(openai.BadRequestError) as exc_info:
|
||||
await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=invalid_audio,
|
||||
language="en",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "Invalid or unsupported audio file" in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_endpoints(whisper_client):
|
||||
# text to text model
|
||||
with pytest.raises(openai.NotFoundError):
|
||||
await whisper_client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "system", "content": "You are a helpful assistant."}],
|
||||
)
|
||||
|
||||
with pytest.raises(openai.NotFoundError):
|
||||
await whisper_client.completions.create(model=MODEL_NAME, prompt="Hello")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_response(winning_call, whisper_client):
|
||||
transcription = ""
|
||||
res_no_stream = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
response_format="json",
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
)
|
||||
res = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
timeout=30,
|
||||
)
|
||||
# Reconstruct from chunks and validate
|
||||
async for chunk in res:
|
||||
text = chunk.choices[0]["delta"]["content"]
|
||||
transcription += text
|
||||
|
||||
assert transcription == res_no_stream.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_options(winning_call, whisper_client):
|
||||
res = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
extra_body=dict(stream_include_usage=True, stream_continuous_usage_stats=True),
|
||||
timeout=30,
|
||||
)
|
||||
final = False
|
||||
continuous = True
|
||||
async for chunk in res:
|
||||
if not len(chunk.choices):
|
||||
# final usage sent
|
||||
final = True
|
||||
else:
|
||||
continuous = continuous and hasattr(chunk, "usage")
|
||||
assert final and continuous
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sampling_params(mary_had_lamb, whisper_client):
|
||||
"""
|
||||
Compare sampling with params and greedy sampling to assert results
|
||||
are different when extreme sampling parameters values are picked.
|
||||
"""
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
temperature=0.8,
|
||||
extra_body=dict(
|
||||
seed=42,
|
||||
repetition_penalty=1.9,
|
||||
top_k=12,
|
||||
top_p=0.4,
|
||||
min_p=0.5,
|
||||
frequency_penalty=1.8,
|
||||
presence_penalty=2.0,
|
||||
),
|
||||
)
|
||||
|
||||
greedy_transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=dict(seed=42),
|
||||
)
|
||||
|
||||
assert greedy_transcription.text != transcription.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_prompt(mary_had_lamb, whisper_client):
|
||||
prompt = "This is a speech, recorded in a phonograph."
|
||||
# Prompts should not omit the part of original prompt while transcribing.
|
||||
prefix = "The first words I spoke in the original phonograph"
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
out = json.loads(transcription)["text"]
|
||||
assert prefix in out
|
||||
transcription_wprompt = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
prompt=prompt,
|
||||
temperature=0.0,
|
||||
)
|
||||
out_prompt = json.loads(transcription_wprompt)["text"]
|
||||
assert prefix in out_prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_with_timestamp(mary_had_lamb, whisper_client):
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="verbose_json",
|
||||
temperature=0.0,
|
||||
)
|
||||
assert transcription.segments is not None
|
||||
assert len(transcription.segments) > 0
|
||||
assert transcription.segments[0].avg_logprob is not None
|
||||
assert transcription.segments[0].compression_ratio is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_with_max_tokens(whisper_client, mary_had_lamb):
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body={"max_completion_tokens": 1},
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tok = AutoTokenizer.from_pretrained(MODEL_NAME)
|
||||
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
|
||||
assert len(out_tokens) == 1
|
||||
# max_completion_tokens > max_model_len
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body={"max_completion_tokens": int(1e6)},
|
||||
)
|
||||
out = json.loads(transcription)
|
||||
out_text = out["text"]
|
||||
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
|
||||
assert len(out_tokens) < 450 # ~Whisper max output len
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("fixture_name", "expected_lang", "expected_text"),
|
||||
[
|
||||
("mary_had_lamb", "en", ["Mary had a little lamb"]),
|
||||
("foscolo", "it", ["zacinto", "sacre"]),
|
||||
],
|
||||
ids=["english", "italian"],
|
||||
)
|
||||
async def test_language_auto_detect(
|
||||
whisper_client, fixture_name, expected_lang, expected_text, request
|
||||
):
|
||||
"""Auto-detect language when no language param is provided."""
|
||||
audio_file = request.getfixturevalue(fixture_name)
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=audio_file,
|
||||
response_format="verbose_json",
|
||||
temperature=0.0,
|
||||
)
|
||||
assert transcription.language == expected_lang
|
||||
text_lower = transcription.text.lower()
|
||||
assert any(word.lower() in text_lower for word in expected_text), (
|
||||
f"Expected {expected_lang} text but got: {transcription.text}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whisper_beam_search_single_beam(mary_had_lamb, whisper_client):
|
||||
"""Test beam search with encoder-decoder model (Whisper) on transcriptions with
|
||||
one beam aligns with greedy decoding.
|
||||
"""
|
||||
beam_transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body=dict(
|
||||
use_beam_search=True,
|
||||
n=1,
|
||||
),
|
||||
)
|
||||
|
||||
greedy_transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
greedy_res = json.loads(greedy_transcription)["text"]
|
||||
beam_res = json.loads(beam_transcription)["text"]
|
||||
assert greedy_res == beam_res
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whisper_beam_search_multibeam(mary_had_lamb, whisper_client):
|
||||
"""Test n>1 for beam search returns one transcription (best beam)."""
|
||||
transcription = await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=mary_had_lamb,
|
||||
language="en",
|
||||
response_format="text",
|
||||
temperature=0.0,
|
||||
extra_body=dict(
|
||||
use_beam_search=True,
|
||||
n=2,
|
||||
),
|
||||
)
|
||||
|
||||
result = json.loads(transcription)
|
||||
|
||||
text = result["text"]
|
||||
|
||||
assert text is not None
|
||||
assert len(text) > 0
|
||||
assert "mary had a little lamb" in text.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_with_beams_raises(winning_call, whisper_client):
|
||||
"""Test that stream=True + beam search raises bad request for now."""
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await whisper_client.audio.transcriptions.create(
|
||||
model=MODEL_NAME,
|
||||
file=winning_call,
|
||||
language="en",
|
||||
stream=True,
|
||||
extra_body=dict(
|
||||
use_beam_search=True,
|
||||
n=2,
|
||||
),
|
||||
)
|
||||
Reference in New Issue
Block a user