From a37453211120bfd7b94042d84f519918d0a3698f Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sun, 11 Jan 2026 21:01:12 +0800 Subject: [PATCH] [CI/Build] Separate out flaky responses API tests (#32110) Signed-off-by: DarkLight1337 --- .buildkite/test-amd.yaml | 18 ++++++++++++++++-- .buildkite/test-pipeline.yaml | 14 +++++++++++++- .buildkite/test_areas/entrypoints.yaml | 11 +++++++++-- tests/entrypoints/openai/responses/__init__.py | 0 .../test_errors.py} | 0 .../test_function_call_parsing.py} | 0 .../test_harmony.py} | 2 +- .../test_mcp_tools.py} | 2 +- .../test_parsable_context.py} | 2 +- .../test_simple.py} | 2 +- 10 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 tests/entrypoints/openai/responses/__init__.py rename tests/entrypoints/openai/{test_responses_error.py => responses/test_errors.py} (100%) rename tests/entrypoints/openai/{test_responses_function_call_parsing.py => responses/test_function_call_parsing.py} (100%) rename tests/entrypoints/openai/{test_response_api_with_harmony.py => responses/test_harmony.py} (99%) rename tests/entrypoints/openai/{test_response_api_mcp_tools.py => responses/test_mcp_tools.py} (99%) rename tests/entrypoints/openai/{test_response_api_parsable_context.py => responses/test_parsable_context.py} (99%) rename tests/entrypoints/openai/{test_response_api_simple.py => responses/test_simple.py} (99%) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 5eadc6bd8f..1fb8c80b25 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -162,8 +162,7 @@ steps: - tests/entrypoints/test_chat_utils commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/test_vision_embeds.py - - pytest -v -s entrypoints/openai/test_vision_embeds.py + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses - pytest -v -s entrypoints/test_chat_utils.py - label: Entrypoints Integration Test (API Server 2) @@ -200,6 +199,21 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/pooling +- label: Entrypoints Integration Test (Responses API) + timeout_in_minutes: 50 + mirror_hardwares: [amdexperimental] + agent_pool: mi325_1 + # grade: Blocking + working_dir: "/vllm-workspace/tests" + fast_check: true + torch_nightly: true + source_file_dependencies: + - vllm/ + - tests/entrypoints/openai/responses + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/openai/responses + - label: Distributed Tests (4 GPUs) # 35min timeout_in_minutes: 50 mirror_hardwares: [amdexperimental] diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 4e004e3479..1c7a5ca368 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -144,7 +144,7 @@ steps: - tests/entrypoints/test_chat_utils commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses - pytest -v -s entrypoints/test_chat_utils.py - label: Entrypoints Integration Test (API Server 2) @@ -177,6 +177,18 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/pooling +- label: Entrypoints Integration Test (Responses API) + timeout_in_minutes: 50 + mirror_hardwares: [amdexperimental] + working_dir: "/vllm-workspace/tests" + fast_check: true + torch_nightly: true + source_file_dependencies: + - vllm/ + - tests/entrypoints/openai/responses + commands: + - pytest -v -s entrypoints/openai/responses + - label: Distributed Tests (4 GPUs) # 35min timeout_in_minutes: 50 mirror_hardwares: [amdexperimental] diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 5b16ea9c1a..8e02d9f60b 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -34,10 +34,9 @@ steps: - tests/entrypoints/test_chat_utils commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses - pytest -v -s entrypoints/test_chat_utils.py - - label: Entrypoints Integration (API Server 2) timeout_in_minutes: 130 working_dir: "/vllm-workspace/tests" @@ -64,6 +63,14 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/pooling +- label: Entrypoints Integration (Responses API) + timeout_in_minutes: 50 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/openai/responses + commands: + - pytest -v -s entrypoints/openai/responses - label: Entrypoints V1 timeout_in_minutes: 50 diff --git a/tests/entrypoints/openai/responses/__init__.py b/tests/entrypoints/openai/responses/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/entrypoints/openai/test_responses_error.py b/tests/entrypoints/openai/responses/test_errors.py similarity index 100% rename from tests/entrypoints/openai/test_responses_error.py rename to tests/entrypoints/openai/responses/test_errors.py diff --git a/tests/entrypoints/openai/test_responses_function_call_parsing.py b/tests/entrypoints/openai/responses/test_function_call_parsing.py similarity index 100% rename from tests/entrypoints/openai/test_responses_function_call_parsing.py rename to tests/entrypoints/openai/responses/test_function_call_parsing.py diff --git a/tests/entrypoints/openai/test_response_api_with_harmony.py b/tests/entrypoints/openai/responses/test_harmony.py similarity index 99% rename from tests/entrypoints/openai/test_response_api_with_harmony.py rename to tests/entrypoints/openai/responses/test_harmony.py index dd18e5f80f..d2a32e3ee0 100644 --- a/tests/entrypoints/openai/test_response_api_with_harmony.py +++ b/tests/entrypoints/openai/responses/test_harmony.py @@ -12,7 +12,7 @@ from openai_harmony import ( Message, ) -from ...utils import RemoteOpenAIServer +from ....utils import RemoteOpenAIServer MODEL_NAME = "openai/gpt-oss-20b" diff --git a/tests/entrypoints/openai/test_response_api_mcp_tools.py b/tests/entrypoints/openai/responses/test_mcp_tools.py similarity index 99% rename from tests/entrypoints/openai/test_response_api_mcp_tools.py rename to tests/entrypoints/openai/responses/test_mcp_tools.py index a66b064ca5..b95a417cf6 100644 --- a/tests/entrypoints/openai/test_response_api_mcp_tools.py +++ b/tests/entrypoints/openai/responses/test_mcp_tools.py @@ -9,7 +9,7 @@ from openai_harmony import ToolDescription, ToolNamespaceConfig from vllm.entrypoints.tool_server import MCPToolServer -from ...utils import RemoteOpenAIServer +from ....utils import RemoteOpenAIServer MODEL_NAME = "openai/gpt-oss-20b" diff --git a/tests/entrypoints/openai/test_response_api_parsable_context.py b/tests/entrypoints/openai/responses/test_parsable_context.py similarity index 99% rename from tests/entrypoints/openai/test_response_api_parsable_context.py rename to tests/entrypoints/openai/responses/test_parsable_context.py index 1e2fd37516..c1f0f435b8 100644 --- a/tests/entrypoints/openai/test_response_api_parsable_context.py +++ b/tests/entrypoints/openai/responses/test_parsable_context.py @@ -8,7 +8,7 @@ import pytest import pytest_asyncio from openai import OpenAI -from ...utils import RemoteOpenAIServer +from ....utils import RemoteOpenAIServer MODEL_NAME = "Qwen/Qwen3-8B" diff --git a/tests/entrypoints/openai/test_response_api_simple.py b/tests/entrypoints/openai/responses/test_simple.py similarity index 99% rename from tests/entrypoints/openai/test_response_api_simple.py rename to tests/entrypoints/openai/responses/test_simple.py index e39e6ab7c6..30423788bf 100644 --- a/tests/entrypoints/openai/test_response_api_simple.py +++ b/tests/entrypoints/openai/responses/test_simple.py @@ -6,7 +6,7 @@ import pytest import pytest_asyncio from openai import OpenAI -from ...utils import RemoteOpenAIServer +from ....utils import RemoteOpenAIServer MODEL_NAME = "Qwen/Qwen3-8B"