From ac87549cbdff37f06c93f96f94005331772f2297 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 26 Jul 2026 22:12:45 -0500 Subject: [PATCH] [CI][ROCm] Reduce V1 attention test runtime (#49916) Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 26 ++----- tests/v1/attention/test_mla_backends.py | 91 ++++++++++++++++--------- 2 files changed, 66 insertions(+), 51 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index b67acbc40b..ebd5d79cce 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -364,22 +364,6 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" -- label: V1 attention (H100-MI250) # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - agent_pool: mi250_1 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/config/attention.py - - vllm/model_executor/layers/attention - - vllm/v1/attention - - tests/v1/attention - - vllm/_aiter_ops.py - - vllm/envs.py - - vllm/platforms/rocm.py - commands: - - pytest -v -s v1/attention - - label: V1 others (CPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] @@ -2694,11 +2678,12 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/kv_connector/extract_hidden_states_integration -- label: V1 attention (H100-MI300) # TBD +- label: V1 attention (H100-MI300) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 + parallelism: 2 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2710,7 +2695,7 @@ steps: - vllm/envs.py - vllm/platforms/rocm.py commands: - - pytest -v -s v1/attention + - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: V1 Core + KV + Metrics # TBD timeout_in_minutes: 180 @@ -3676,11 +3661,12 @@ steps: #------------------------------------------------------------ mi355 ยท v1 -------------------------------------------------------------# -- label: V1 attention (B200-MI355) # TBD +- label: V1 attention (B200-MI355) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 + parallelism: 2 working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/config/attention.py @@ -3691,7 +3677,7 @@ steps: - vllm/envs.py - vllm/platforms/rocm.py commands: - - pytest -v -s v1/attention + - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: V1 Core + KV + Metrics # TBD timeout_in_minutes: 180 diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index 7de61b32a7..c520f22e7e 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -40,6 +40,10 @@ from vllm.v1.attention.backends.mla.prefill import ( MLAPrefillBackendEnum, get_mla_prefill_backend, ) +from vllm.v1.attention.backends.mla.prefill.base import MLADimensions +from vllm.v1.attention.backends.mla.prefill.selector import ( + MLAPrefillSelectorConfig, +) from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported from vllm.v1.kv_cache_interface import ( @@ -145,14 +149,67 @@ def test_mla_post_load_preserves_runtime_weight_addresses(monkeypatch): torch.testing.assert_close(layer.W_UK_T, old_w_uk_t + 100) -# Filtered per-test via validate_configuration (capability/deps/dims). +# Validate parameter combinations during collection, before GPU fixtures run. PREFILL_BACKENDS_TO_TEST = [ + MLAPrefillBackendEnum.ROCM_AITER_FA, MLAPrefillBackendEnum.FLASH_ATTN, MLAPrefillBackendEnum.FLASHINFER, MLAPrefillBackendEnum.TRTLLM_RAGGED, MLAPrefillBackendEnum.TOKENSPEED_MLA, ] +MLA_DIMENSIONS_TO_TEST = [ + ("deepseek", 128, 128), + ("glm", 192, 256), +] + + +def _prefill_backend_dimension_params(): + device_capability = current_platform.get_device_capability() + params = [] + for prefill_backend in PREFILL_BACKENDS_TO_TEST: + for dimensions_id, qk_nope_head_dim, v_head_dim in MLA_DIMENSIONS_TO_TEST: + if device_capability is None: + invalid_reasons = ["device capability unavailable"] + else: + try: + invalid_reasons = ( + prefill_backend.get_class().validate_configuration( + device_capability, + MLAPrefillSelectorConfig( + dtype=torch.bfloat16, + mla_dimensions=MLADimensions( + qk_nope_head_dim=qk_nope_head_dim, + qk_rope_head_dim=64, + v_head_dim=v_head_dim, + ), + ), + ) + ) + except ImportError: + invalid_reasons = ["ImportError"] + + marks = [] + if invalid_reasons: + marks.append( + pytest.mark.skip( + reason=( + f"Prefill backend {prefill_backend.name} unavailable: " + f"{invalid_reasons}" + ) + ) + ) + params.append( + pytest.param( + prefill_backend, + qk_nope_head_dim, + v_head_dim, + id=f"{dimensions_id}-{prefill_backend}", + marks=marks, + ) + ) + return params + SPEC_DECODE_BACKENDS = [] for backend in BACKENDS_TO_TEST: @@ -1098,11 +1155,9 @@ def run_attention_backend( @pytest.mark.parametrize("tensor_parallel_size", [1, 4, 8, 16]) @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"]) @pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)]) -@pytest.mark.parametrize("prefill_backend", PREFILL_BACKENDS_TO_TEST) @pytest.mark.parametrize( - ("qk_nope_head_dim", "v_head_dim"), - [(128, 128), (192, 256)], - ids=["deepseek", "glm"], + ("prefill_backend", "qk_nope_head_dim", "v_head_dim"), + _prefill_backend_dimension_params(), ) def test_backend_correctness( default_vllm_config, @@ -1153,32 +1208,6 @@ def test_backend_correctness( if not backends_to_test: pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}") - # Skip prefill backends that can't satisfy capability/deps/dimension constraints. - from vllm.v1.attention.backends.mla.prefill.base import MLADimensions - from vllm.v1.attention.backends.mla.prefill.selector import ( - MLAPrefillSelectorConfig, - ) - - try: - prefill_invalid_reasons = prefill_backend.get_class().validate_configuration( - current_platform.get_device_capability(), - MLAPrefillSelectorConfig( - dtype=torch.bfloat16, - mla_dimensions=MLADimensions( - qk_nope_head_dim=qk_nope_head_dim, - qk_rope_head_dim=64, - v_head_dim=v_head_dim, - ), - ), - ) - except ImportError: - prefill_invalid_reasons = ["ImportError"] - if prefill_invalid_reasons: - pytest.skip( - f"Prefill backend {prefill_backend.name} unavailable: " - f"{prefill_invalid_reasons}" - ) - batch_spec = BATCH_SPECS[batch_spec_name] is_spec_decode_test = batch_spec_name.startswith("spec_decode") unique_block_sizes = sorted(set(BACKEND_BLOCK_SIZES[b] for b in backends_to_test))