diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 6ed1ae6338..76207e5b36 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1322,7 +1322,6 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - - pytest -v -s entrypoints/openai/speech_to_text/ - pytest -v -s entrypoints/test_chat_utils.py - label: Entrypoints Integration (API Server openai - Part 3) # TBD @@ -1338,7 +1337,21 @@ steps: - tests/entrypoints/test_chat_utils commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py + +- label: Entrypoints Integration (Speech to Text) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + agent_pool: mi300_1 + fast_check: true + torch_nightly: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/speech_to_text + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/speech_to_text - label: Entrypoints Integration (LLM) # TBD timeout_in_minutes: 180 @@ -2768,7 +2781,6 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - - pytest -v -s entrypoints/openai/speech_to_text/ - pytest -v -s entrypoints/test_chat_utils.py - label: Entrypoints Integration (API Server openai - Part 3) # TBD @@ -2784,7 +2796,21 @@ steps: - tests/entrypoints/test_chat_utils commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py + +- label: Entrypoints Integration (Speech to Text) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] + agent_pool: mi355_1 + fast_check: true + torch_nightly: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/speech_to_text + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/speech_to_text - label: Entrypoints Integration (Pooling) # TBD timeout_in_minutes: 180 diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index ba92d3a3ae..f9ddf26030 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -11,7 +11,7 @@ steps: - tests/entrypoints/ commands: - pytest -v -s entrypoints/openai/tool_parsers - - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling + - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text - label: Entrypoints Integration (LLM) key: entrypoints-integration-llm @@ -44,7 +44,6 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py - - label: Entrypoints Integration (API Server openai - Part 2) key: entrypoints-integration-api-server-openai-part-2 timeout_in_minutes: 50 @@ -55,7 +54,6 @@ steps: - tests/entrypoints/test_chat_utils commands: - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - - pytest -v -s entrypoints/openai/speech_to_text/ - pytest -v -s entrypoints/test_chat_utils.py - label: Entrypoints Integration (API Server openai - Part 3) @@ -69,7 +67,7 @@ steps: - tests/entrypoints/test_chat_utils commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py - label: Entrypoints Integration (API Server 2) key: entrypoints-integration-api-server-2 @@ -86,6 +84,17 @@ steps: - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc - pytest -v -s tool_use +- label: Entrypoints Integration (Speech to Text) + key: entrypoints-integration-speech_to_text + timeout_in_minutes: 50 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/speech_to_text + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/speech_to_text + - label: Entrypoints Integration (Pooling) key: entrypoints-integration-pooling timeout_in_minutes: 50 @@ -115,5 +124,5 @@ steps: - csrc/ - vllm/entrypoints/openai/ - vllm/model_executor/models/whisper.py - commands: # LMEval+Transcription WER check + commands: # LMEval - pytest -s entrypoints/openai/correctness/ diff --git a/pyproject.toml b/pyproject.toml index 6a0ae2fea6..0641d5aca2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,7 +123,8 @@ python = "./.venv" extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizers_/*", "benchmarks/sonnet.txt", "tests/lora/data/*", "build/*", "examples/pooling/token_embed/*", "tests/models/language/pooling/*", - "vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", "tests/entrypoints/openai/speech_to_text/test_transcription_validation.py", + "vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", + "tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py", "docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html", "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*"] ignore-hidden = false diff --git a/tests/entrypoints/openai/chat_completion/test_enable_force_include_usage.py b/tests/entrypoints/openai/chat_completion/test_enable_force_include_usage.py index 1bc545e864..75ddeb43ab 100644 --- a/tests/entrypoints/openai/chat_completion/test_enable_force_include_usage.py +++ b/tests/entrypoints/openai/chat_completion/test_enable_force_include_usage.py @@ -67,58 +67,3 @@ async def test_chat_with_enable_force_include_usage( chunk.usage.prompt_tokens + chunk.usage.completion_tokens ) last_completion_tokens = chunk.usage.completion_tokens - - -@pytest.fixture(scope="module") -def transcription_server_with_force_include_usage(): - args = [ - # use half precision for speed and memory savings in CI environment - "--dtype", - "bfloat16", - "--max-num-seqs", - "4", - "--enforce-eager", - "--enable-force-include-usage", - "--gpu-memory-utilization", - "0.2", - ] - - with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server: - yield remote_server - - -@pytest_asyncio.fixture -async def transcription_client_with_force_include_usage( - transcription_server_with_force_include_usage, -): - async with ( - transcription_server_with_force_include_usage.get_async_client() as async_client - ): - yield async_client - - -@pytest.mark.asyncio -async def test_transcription_with_enable_force_include_usage( - transcription_client_with_force_include_usage, winning_call -): - res = ( - await transcription_client_with_force_include_usage.audio.transcriptions.create( - model="openai/whisper-large-v3-turbo", - file=winning_call, - language="en", - temperature=0.0, - stream=True, - timeout=30, - ) - ) - - async for chunk in res: - if not len(chunk.choices): - # final usage sent - usage = chunk.usage - assert isinstance(usage, dict) - assert usage["prompt_tokens"] > 0 - assert usage["completion_tokens"] > 0 - assert usage["total_tokens"] > 0 - else: - assert not hasattr(chunk, "usage") diff --git a/tests/entrypoints/openai/realtime/__init__.py b/tests/entrypoints/speech_to_text/__init__.py similarity index 100% rename from tests/entrypoints/openai/realtime/__init__.py rename to tests/entrypoints/speech_to_text/__init__.py diff --git a/tests/entrypoints/openai/conftest.py b/tests/entrypoints/speech_to_text/conftest.py similarity index 100% rename from tests/entrypoints/openai/conftest.py rename to tests/entrypoints/speech_to_text/conftest.py diff --git a/tests/entrypoints/openai/speech_to_text/__init__.py b/tests/entrypoints/speech_to_text/correctness/__init__.py similarity index 100% rename from tests/entrypoints/openai/speech_to_text/__init__.py rename to tests/entrypoints/speech_to_text/correctness/__init__.py diff --git a/tests/entrypoints/openai/correctness/test_transcription_api_correctness.py b/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py similarity index 100% rename from tests/entrypoints/openai/correctness/test_transcription_api_correctness.py rename to tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py diff --git a/tests/entrypoints/speech_to_text/realtime/__init__.py b/tests/entrypoints/speech_to_text/realtime/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/entrypoints/openai/realtime/test_realtime_validation.py b/tests/entrypoints/speech_to_text/realtime/test_realtime_validation.py similarity index 97% rename from tests/entrypoints/openai/realtime/test_realtime_validation.py rename to tests/entrypoints/speech_to_text/realtime/test_realtime_validation.py index e317090fa5..675922f588 100644 --- a/tests/entrypoints/openai/realtime/test_realtime_validation.py +++ b/tests/entrypoints/speech_to_text/realtime/test_realtime_validation.py @@ -10,7 +10,7 @@ import pybase64 as base64 import pytest import websockets -from tests.entrypoints.openai.conftest import add_attention_backend +from tests.entrypoints.speech_to_text.conftest import add_attention_backend from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer from vllm.assets.audio import AudioAsset from vllm.multimodal.media.audio import load_audio @@ -163,6 +163,11 @@ async def test_multi_chunk_streaming( " A little piece of practical poetry. Mary had a little lamb," " it sleeps with quite a flow, and everywhere that Mary went," " the lamb was sure to go." + ) or full_text == ( + " First words I spoke in the original phonograph." + " A little piece of practical poetry. Mary had a little lamb," + " it squeaked with quite a flow, and everywhere that Mary went," + " the lamb was sure to go." ) diff --git a/tests/entrypoints/openai/speech_to_text/test_speech_to_text_cancellation.py b/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py similarity index 100% rename from tests/entrypoints/openai/speech_to_text/test_speech_to_text_cancellation.py rename to tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py diff --git a/tests/entrypoints/speech_to_text/transcription/__init__.py b/tests/entrypoints/speech_to_text/transcription/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/entrypoints/speech_to_text/transcription/test_enable_force_include_usage.py b/tests/entrypoints/speech_to_text/transcription/test_enable_force_include_usage.py new file mode 100644 index 0000000000..0477e7082b --- /dev/null +++ b/tests/entrypoints/speech_to_text/transcription/test_enable_force_include_usage.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest +import pytest_asyncio + +from tests.utils import RemoteOpenAIServer + + +@pytest.fixture(scope="module") +def transcription_server_with_force_include_usage(): + args = [ + # use half precision for speed and memory savings in CI environment + "--dtype", + "bfloat16", + "--max-num-seqs", + "4", + "--enforce-eager", + "--enable-force-include-usage", + "--gpu-memory-utilization", + "0.2", + ] + + with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server: + yield remote_server + + +@pytest_asyncio.fixture +async def transcription_client_with_force_include_usage( + transcription_server_with_force_include_usage, +): + async with ( + transcription_server_with_force_include_usage.get_async_client() as async_client + ): + yield async_client + + +@pytest.mark.asyncio +async def test_transcription_with_enable_force_include_usage( + transcription_client_with_force_include_usage, winning_call +): + res = ( + await transcription_client_with_force_include_usage.audio.transcriptions.create( + model="openai/whisper-large-v3-turbo", + file=winning_call, + language="en", + temperature=0.0, + stream=True, + timeout=30, + ) + ) + + async for chunk in res: + if not len(chunk.choices): + # final usage sent + usage = chunk.usage + assert isinstance(usage, dict) + assert usage["prompt_tokens"] > 0 + assert usage["completion_tokens"] > 0 + assert usage["total_tokens"] > 0 + else: + assert not hasattr(chunk, "usage") diff --git a/tests/entrypoints/openai/speech_to_text/test_transcription_inter_chunk_spacing.py b/tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py similarity index 100% rename from tests/entrypoints/openai/speech_to_text/test_transcription_inter_chunk_spacing.py rename to tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py diff --git a/tests/entrypoints/openai/speech_to_text/test_transcription_validation.py b/tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py similarity index 98% rename from tests/entrypoints/openai/speech_to_text/test_transcription_validation.py rename to tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py index 4ac48699a0..5ea218406b 100644 --- a/tests/entrypoints/openai/speech_to_text/test_transcription_validation.py +++ b/tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py @@ -6,7 +6,7 @@ import json import pytest -from tests.entrypoints.openai.conftest import add_attention_backend +from tests.entrypoints.speech_to_text.conftest import add_attention_backend from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer MISTRAL_FORMAT_ARGS = [ diff --git a/tests/entrypoints/openai/speech_to_text/test_transcription_validation_whisper.py b/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py similarity index 100% rename from tests/entrypoints/openai/speech_to_text/test_transcription_validation_whisper.py rename to tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py diff --git a/tests/entrypoints/speech_to_text/translation/__init__.py b/tests/entrypoints/speech_to_text/translation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/entrypoints/openai/speech_to_text/test_translation_validation.py b/tests/entrypoints/speech_to_text/translation/test_translation_validation.py similarity index 99% rename from tests/entrypoints/openai/speech_to_text/test_translation_validation.py rename to tests/entrypoints/speech_to_text/translation/test_translation_validation.py index a8b17bf343..ed3cff5f1c 100644 --- a/tests/entrypoints/openai/speech_to_text/test_translation_validation.py +++ b/tests/entrypoints/speech_to_text/translation/test_translation_validation.py @@ -13,7 +13,7 @@ import pytest import pytest_asyncio import soundfile as sf -from tests.entrypoints.openai.conftest import add_attention_backend +from tests.entrypoints.speech_to_text.conftest import add_attention_backend from tests.utils import RemoteOpenAIServer from vllm.logger import init_logger from vllm.multimodal.media.audio import load_audio