From 91740ca5eaedbcdd960c3629206087d20fb43a00 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 6 May 2026 00:05:20 -0500 Subject: [PATCH] [ROCm][CI] Refine gating tests (#37243) Signed-off-by: Andreas Karatzas --- .../scripts/hardware_ci/run-amd-test.sh | 14 +++++++++--- .buildkite/test-amd.yaml | 11 +++++++++- .buildkite/test_areas/engine.yaml | 10 --------- .buildkite/test_areas/entrypoints.yaml | 15 +++++-------- .buildkite/test_areas/misc.yaml | 12 +--------- .buildkite/test_areas/models_basic.yaml | 6 ----- .buildkite/test_areas/models_language.yaml | 22 +++++++------------ .buildkite/test_areas/models_multimodal.yaml | 13 ++++------- .buildkite/test_areas/samplers.yaml | 2 +- 9 files changed, 40 insertions(+), 65 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 703a7d7532..7e8ddb12ec 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -378,9 +378,11 @@ HF_MOUNT="/root/.cache/huggingface" # double-quotes will have been stripped by the calling shell. if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then commands="${VLLM_TEST_COMMANDS}" + commands_source="env" echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)" else commands="$*" + commands_source="argv" if [[ -z "$commands" ]]; then echo "Error: No test commands provided." >&2 echo "Usage:" >&2 @@ -397,9 +399,15 @@ fi echo "Raw commands: $commands" -# Fix quoting before ROCm overrides (so overrides see correct structure) -commands=$(re_quote_pytest_markers "$commands") -echo "After re-quoting: $commands" +# Only try to repair stripped pytest -m/-k quoting in legacy argv mode. +# VLLM_TEST_COMMANDS preserves inner quoting already, and re-quoting that path +# can corrupt embedded echo strings or otherwise well-formed shell fragments. +if [[ "$commands_source" == "argv" ]]; then + commands=$(re_quote_pytest_markers "$commands") + echo "After re-quoting: $commands" +else + echo "Skipping re-quoting for VLLM_TEST_COMMANDS input" +fi commands=$(apply_rocm_test_overrides "$commands") echo "Final commands: $commands" diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 844dbe639b..5b3eb4f79c 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -621,6 +621,7 @@ steps: mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 torch_nightly: true + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -1803,6 +1804,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -1844,6 +1846,7 @@ steps: mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 torch_nightly: true + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -2204,7 +2207,6 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 - optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -2628,6 +2630,7 @@ steps: agent_pool: mi325_1 torch_nightly: true parallelism: 2 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -2653,6 +2656,7 @@ steps: mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] agent_pool: mi325_1 torch_nightly: true + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -2749,6 +2753,7 @@ steps: agent_pool: mi355_1 fast_check: true torch_nightly: true + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -2764,6 +2769,7 @@ steps: agent_pool: mi355_1 fast_check: true torch_nightly: true + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -3060,6 +3066,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -3259,6 +3266,7 @@ steps: timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -3284,6 +3292,7 @@ steps: timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index b83eaf669a..cf0f028255 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -57,11 +57,6 @@ steps: commands: # Only run tests that need exactly 2 GPUs - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism" - mirror: - amd: - device: mi325_2 - depends_on: - - image-build-amd - label: V1 e2e (4 GPUs) key: v1-e2e-4-gpus @@ -74,11 +69,6 @@ steps: commands: # Only run tests that need 4 GPUs - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy" - mirror: - amd: - device: mi325_4 - depends_on: - - image-build-amd - label: V1 e2e (4xH100) key: v1-e2e-4xh100 diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 4908df1e4e..ba92d3a3ae 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -26,6 +26,11 @@ steps: - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests + mirror: + amd: + device: mi300_1 + depends_on: + - image-build-amd - label: Entrypoints Integration (API Server openai - Part 1) key: entrypoints-integration-api-server-openai-part-1 @@ -38,11 +43,6 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - label: Entrypoints Integration (API Server openai - Part 2) @@ -57,11 +57,6 @@ steps: - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - pytest -v -s entrypoints/openai/speech_to_text/ - pytest -v -s entrypoints/test_chat_utils.py - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - label: Entrypoints Integration (API Server openai - Part 3) key: entrypoints-integration-api-server-openai-part-3 diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index fffea26517..0e6e709598 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -12,11 +12,6 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn # TODO: create another `optional` test group for slow tests - pytest -v -s -m 'not slow_test' v1/spec_decode - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - label: V1 Sample + Logits key: v1-sample-logits @@ -38,7 +33,7 @@ steps: - pytest -v -s v1/test_outputs.py mirror: amd: - device: mi325_1 + device: mi300_1 depends_on: - image-build-amd @@ -67,11 +62,6 @@ steps: # Integration test for streaming correctness (requires special branch). - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - label: V1 Others (CPU) key: v1-others-cpu diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index ab597f960e..8fca203de4 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -42,12 +42,6 @@ steps: - tests/models/test_registry.py commands: - pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - - label: Basic Models Test (Other CPU) # 5min key: basic-models-test-other-cpu diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 64f8fc809d..b560c5a476 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -48,6 +48,14 @@ steps: parallelism: 2 mirror: torch_nightly: {} + amd: + device: mi300_1 + depends_on: + - image-build-amd + commands: + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' + - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: Language Models Test (Extended Generation) # 80min key: language-models-test-extended-generation @@ -62,15 +70,6 @@ steps: - uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0' - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - commands: - - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' - - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' - label: Language Models Test (PPL) key: language-models-test-ppl @@ -92,11 +91,6 @@ steps: - tests/models/language/pooling commands: - pytest -v -s models/language/pooling -m 'not core_model' - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - label: Language Models Test (MTEB) key: language-models-test-mteb diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 857cc68f2a..1f66393df8 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -15,7 +15,7 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model mirror: amd: - device: mi325_1 + device: mi300_1 depends_on: - image-build-amd @@ -33,7 +33,7 @@ steps: - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model mirror: amd: - device: mi325_1 + device: mi300_1 depends_on: - image-build-amd @@ -49,7 +49,7 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model mirror: amd: - device: mi325_1 + device: mi300_1 depends_on: - image-build-amd @@ -64,11 +64,6 @@ steps: - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing - pytest models/multimodal/generation/test_memory_leak.py -m core_model - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - label: Multi-Modal Processor (CPU) key: multi-modal-processor-cpu @@ -120,7 +115,7 @@ steps: - pytest -v -s models/multimodal/test_mapping.py mirror: amd: - device: mi325_1 + device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/samplers.yaml b/.buildkite/test_areas/samplers.yaml index 37f8eaa688..48e9f55571 100644 --- a/.buildkite/test_areas/samplers.yaml +++ b/.buildkite/test_areas/samplers.yaml @@ -17,7 +17,7 @@ steps: - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers mirror: amd: - device: mi325_1 + device: mi250_1 depends_on: - image-build-amd commands: