From e9f167129c09d5330805c2042c16329e9b218c5a Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Thu, 24 Sep 2026 04:01:58 +0200 Subject: [PATCH] [CI][ROCM] Add the Fusion E2E TP2 Quick group on MI355, and the AITER MLA fix it needs (#58369) Signed-off-by: Stefan Koncarevic Signed-off-by: Andreas Karatzas Co-authored-by: Andreas Karatzas Co-authored-by: Andreas Karatzas Co-authored-by: Codex --- .buildkite/test-amd.yaml | 41 ++++++++++++ .buildkite/test_areas/compile.yaml | 41 ++++++++++++ tests/compile/fusions_e2e/models.py | 9 +++ tests/compile/fusions_e2e/test_tp2_ar_rms.py | 65 ++++++++++++++++--- .../test_rocm_aiter_mla_fp8_decode_routing.py | 19 ++++++ .../test_rocm_aiter_mla_mtp_split.py | 4 +- .../attention/backends/mla/rocm_aiter_mla.py | 57 ++++++++++++++-- 7 files changed, 220 insertions(+), 16 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 5b630d773f..a349f8a58e 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3773,6 +3773,47 @@ steps: - pytest -v -s tests/distributed/test_rocm_quick_reduce.py - pytest -v -s tests/distributed/test_quick_all_reduce.py +- label: ":amd: (MI355) Fusion E2E TP2 Quick" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - csrc/quantization/ + - vllm/compilation/ + - vllm/distributed/ + - vllm/model_executor/ + - vllm/v1/attention/ + - tests/compile/fusions_e2e/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + not +quant_fp8 and not +rms_norm and Llama-3 and + ROCM_AITER_UNIFIED_ATTN' + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + +quant_fp8 and +rms_norm and qwen3 and + ROCM_AITER_UNIFIED_ATTN' + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + not +quant_fp8 and not +rms_norm and DeepSeek-Coder and + ROCM_AITER_MLA' + #-------------------------------------------------------- mi355 ยท entrypoints --------------------------------------------------------# - label: ":amd: (MI355) Entrypoints Integration (API Server) %N" diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index 6a7a6f909f..2f314aa138 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -206,6 +206,47 @@ steps: # Run all models and attn backends but only Inductor partition and native custom ops - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))" - pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))" + mirror: + amd: + label: ":amd: (MI355) Fusion E2E TP2 Quick" + dind: false + device: mi355_2 + timeout_in_minutes: 60 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/ + source_file_dependencies: + - csrc/quantization/ + - vllm/compilation/ + - vllm/distributed/ + - vllm/model_executor/ + - vllm/v1/attention/ + - tests/compile/fusions_e2e/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + not +quant_fp8 and not +rms_norm and Llama-3 and + ROCM_AITER_UNIFIED_ATTN' + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + +quant_fp8 and +rms_norm and qwen3 and + ROCM_AITER_UNIFIED_ATTN' + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + not +quant_fp8 and not +rms_norm and DeepSeek-Coder and + ROCM_AITER_MLA' - label: ":nvidia: (H100) Fusion E2E TP2 AR-RMS Config Sweep" key: fusion-e2e-tp2-ar-rms-config-sweep-h100 diff --git a/tests/compile/fusions_e2e/models.py b/tests/compile/fusions_e2e/models.py index 7879cbe650..eadf1739fd 100644 --- a/tests/compile/fusions_e2e/models.py +++ b/tests/compile/fusions_e2e/models.py @@ -44,6 +44,15 @@ ROCM_AITER_UNIFIED_ATTN = pytest.param( ), ) +ROCM_AITER_MLA_ATTN = pytest.param( + AttentionBackendCase(backend=AttentionBackendEnum.ROCM_AITER_MLA), + id="ROCM_AITER_MLA", + marks=pytest.mark.skipif( + not is_aiter_found_and_supported(), + reason="ROCM_AITER_MLA only for AMD when AITER is installed", + ), +) + FLASHINFER_MLA_ATTN = pytest.param( AttentionBackendCase(backend=AttentionBackendEnum.FLASHINFER_MLA), id="FLASHINFER_MLA", diff --git a/tests/compile/fusions_e2e/test_tp2_ar_rms.py b/tests/compile/fusions_e2e/test_tp2_ar_rms.py index 4b3db06bc4..f35bc1bd9c 100644 --- a/tests/compile/fusions_e2e/test_tp2_ar_rms.py +++ b/tests/compile/fusions_e2e/test_tp2_ar_rms.py @@ -19,6 +19,7 @@ from .common import ( from .models import ( FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, + ROCM_AITER_MLA_ATTN, ROCM_AITER_UNIFIED_ATTN, ROCM_ATTN, TRITON_ATTN, @@ -40,25 +41,35 @@ pytestmark = pytest.mark.skipif( ) -@multi_gpu_test(num_gpus=2) -@pytest.mark.parametrize( - "model_name, matches_fn, model_kwargs, hf_overrides", - # qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet supported +# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet +# supported. ROCm carries only the payloads its two backends support. +TP2_FP8_MODELS = ( [ llama3_8b_fp8, llama4_scout_fp8, qwen3_a3b_fp8, deepseek_coder_v2_lite_fp8, deepseek_v3_fp8, - ], + ] + if current_platform.is_cuda() + else [llama3_8b_fp8, qwen3_a3b_fp8, deepseek_coder_v2_lite_fp8] ) + +TP2_FP8_ATTN_BACKENDS = ( + [TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN] + if current_platform.is_cuda() + else [ROCM_AITER_UNIFIED_ATTN, ROCM_AITER_MLA_ATTN] +) + + +@multi_gpu_test(num_gpus=2) @pytest.mark.parametrize( - "attn_backend", [TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN] + "model_name, matches_fn, model_kwargs, hf_overrides", TP2_FP8_MODELS ) +@pytest.mark.parametrize("attn_backend", TP2_FP8_ATTN_BACKENDS) @pytest.mark.parametrize("n_layers", [4]) @pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm")) @pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION) -@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA") def test_tp2_ar_rms_fp8_fusions( model_name: str, matches_fn: Callable[[int], Matches], @@ -69,10 +80,27 @@ def test_tp2_ar_rms_fp8_fusions( custom_ops: str, inductor_graph_partition: bool, run_e2e_fusion_test, + monkeypatch, ): matches = matches_fn(n_layers) - block_fp8 = "qwen" in model_name.lower() or "deepseek" in model_name.lower() + model_name_lower = model_name.lower() + # DeepSeek-Coder on AITER MLA is the one static-FP8 payload here, validated + # against the native QuantFP8 pattern; every other DeepSeek stays group + # quantized as on CUDA. + rocm_static_fp8_mla = ( + current_platform.is_rocm() + and "deepseek-coder" in model_name_lower + and attn_backend.backend.name == "ROCM_AITER_MLA" + ) + block_fp8 = "qwen" in model_name_lower or "deepseek" in model_name_lower + if rocm_static_fp8_mla: + block_fp8 = False + if custom_ops != "-quant_fp8,-rms_norm": + pytest.skip("ROCm static-FP8 MLA parity uses native quant and RMSNorm") + # AITER's MoE GEMM rejects this reduced dummy model shape; keep AITER + # for MLA and the all-reduce, and let the MoE run on Triton. + monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "0") if block_fp8 and "-quant_fp8" in custom_ops: # This is why config forces +quant_fp8 by default pytest.skip("native QuantFP8 matching not supported for group quant") @@ -83,6 +111,13 @@ def test_tp2_ar_rms_fp8_fusions( model_kwargs["max_model_len"] = 1024 model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False} model_kwargs["disable_custom_all_reduce"] = False + if current_platform.is_rocm() and "llama-3" in model_name_lower: + model_kwargs.update( + gpu_memory_utilization=0.01, + kv_cache_memory_bytes=64 * 1024**2, + max_num_batched_tokens=1024, + max_num_seqs=32, + ) compilation_config = dict( use_inductor_graph_partition=inductor_graph_partition, @@ -103,6 +138,19 @@ def test_tp2_ar_rms_fp8_fusions( "ar_rms_fusion", ] + use_aiter = current_platform.is_rocm() + if use_aiter: + matches = matches._replace(aiter_ar_rms_fusion=matches.ar_rms_fusion) + # The AITER all-reduce pass registers the AR+RMS(+group-quant) patterns + # before its AR+RMS-only ones, so the standalone AITER RMS-quant pass + # does not own the post-AR sites here. ROCm's contract is native + # QK-norm+RoPE and attention-output quant fusion plus AITER AR+RMS. + matches_check = [ + "norm_rope_fusion", + "attn_quant_fusion", + "aiter_ar_rms_fusion", + ] + run_e2e_fusion_test( model_name, matches, @@ -111,6 +159,7 @@ def test_tp2_ar_rms_fp8_fusions( compilation_config, matches_check, tp_size=2, + use_aiter=use_aiter, ) diff --git a/tests/v1/attention/test_rocm_aiter_mla_fp8_decode_routing.py b/tests/v1/attention/test_rocm_aiter_mla_fp8_decode_routing.py index a86c7e617b..e6390ce36e 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_fp8_decode_routing.py +++ b/tests/v1/attention/test_rocm_aiter_mla_fp8_decode_routing.py @@ -151,6 +151,25 @@ def test_decode_and_verify_are_disjoint(kv_cache_dtype, num_heads): ) +@pytest.mark.parametrize("kv_cache_dtype", UNQUANTIZED_DTYPES) +def test_oversized_kv_cache_refuses_gluon(gluon_available, kv_cache_dtype): + """Past 2 GiB per layer Gluon drops its KV bounds mask and reads out of range. + + ``None`` is the profiling case, before the cache is sized: nothing to + refuse yet, so the usual route stands. + """ + bound = rocm_aiter_mla._GLUON_MAX_KV_CACHE_BYTES + for kv_cache_bytes in (None, bound): + assert AiterMLAHelper.use_gluon_decode(8, 1, kv_cache_dtype, kv_cache_bytes) + assert AiterMLAHelper.use_gluon_verify( + 8, 8, kv_cache_dtype, kv_cache_bytes=kv_cache_bytes + ) + assert not AiterMLAHelper.use_gluon_decode(8, 1, kv_cache_dtype, bound + 1) + assert not AiterMLAHelper.use_gluon_verify( + 8, 8, kv_cache_dtype, kv_cache_bytes=bound + 1 + ) + + @pytest.mark.parametrize("num_heads", [1, 2, 3, 5, 6, 7, 8, 9, 12, 15]) def test_padded_query_is_contiguous(num_heads): """asm_mla.cu:805 requires Q.is_contiguous(). diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index 2d8a5c2063..1970dbb54f 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -107,6 +107,8 @@ def _builder( _dcp_verify_buffers=None, _graph_seq_lens=None, _kv_cache_dtype_str=kv_cache_dtype, + # Unsized as during profiling, so the Gluon KV bound stays out of the way. + _kv_cache_bytes=None, paged_kv_last_page_len=torch.ones(max_decode_rows, dtype=torch.int32), paged_kv_indices=torch.empty(1024, dtype=torch.int32), paged_kv_indptr=torch.empty(max_decode_rows + 1, dtype=torch.int32), @@ -585,7 +587,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata( get_num_attention_heads=lambda parallel_config: num_heads, ), scheduler_config=SimpleNamespace(max_num_seqs=2), - cache_config=SimpleNamespace(cache_dtype="fp8_e4m3"), + cache_config=SimpleNamespace(cache_dtype="fp8_e4m3", num_gpu_blocks=None), compilation_config=SimpleNamespace( cudagraph_mode=SimpleNamespace(has_full_cudagraphs=lambda: False), # Empty: the per-layer head-count probe finds no attention layer and diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index 3728a451a7..3fa5f67bf5 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -178,6 +178,30 @@ def _gluon_mla_decode_supported() -> bool: return on_gfx950() +# Past 2 GiB, mla_gluon swaps its masked buffer_load for an unmasked +# global_load that reads outside the cache and aborts the process. Mirrors the +# kernel's own within_2gb bound, so the fallback engages exactly where it does. +_GLUON_MAX_KV_CACHE_BYTES = 1 << 31 + + +def _gluon_kv_cache_in_bounds(kv_cache_bytes: int | None) -> bool: + """Whether this layer's KV cache keeps Gluon on its bounds-checked path. + + The cache is one flat tensor per layer, so the bound disqualifies the layer + outright; no batch shape brings an oversized cache back in range. ``None`` + is the not-yet-sized case during profiling, before any kernel runs. + """ + if kv_cache_bytes is None or kv_cache_bytes <= _GLUON_MAX_KV_CACHE_BYTES: + return True + logger.warning_once( + "KV cache is %.1f GiB per layer, past the 2 GiB where the Gluon MLA " + "kernel drops its KV bounds mask; using the padded ASM decode instead. " + "Lower --gpu-memory-utilization or --max-model-len to get Gluon back.", + kv_cache_bytes / (1 << 30), + ) + return False + + @functools.lru_cache(maxsize=1) def _get_segmented_mla_decode(): """Load AITER's segmented MLA decode with unreduced partial output.""" @@ -564,6 +588,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): # _build_decode needs the cache dtype to pick the decode kernel; keep # the normalized string instead of dropping it at the end of __init__. self._kv_cache_dtype_str = kv_cache_dtype_str + # Sized per layer, matching the flat KV tensor the kernel is handed. + num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks + self._kv_cache_bytes = ( + None + if num_gpu_blocks is None + else num_gpu_blocks * kv_cache_spec.page_size_bytes + ) # MLAAttention quantizes decode Q to FP8 before calling this backend # whenever the KV cache is FP8 and supports_quant_query_input is true. q_dtype = ( @@ -1065,6 +1096,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): self._decode_num_heads, int(max_qo_len), self._kv_cache_dtype_str, + self._kv_cache_bytes, ) use_gluon_verify = AiterMLAHelper.use_gluon_verify( self._decode_num_heads, @@ -1072,6 +1104,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): self._kv_cache_dtype_str, self.dcp_world_size, causal, + self._kv_cache_bytes, ) use_segmented_dcp_verify = ( self._supports_segmented_dcp_verify and max_qo_len > 1 @@ -1481,7 +1514,12 @@ class AiterMLAHelper: return AiterMLAHelper._get_mla_unpadded_heads(num_heads, lse) @staticmethod - def use_gluon_decode(num_heads: int, max_qo_len: int, kv_cache_dtype: str) -> bool: + def use_gluon_decode( + num_heads: int, + max_qo_len: int, + kv_cache_dtype: str, + kv_cache_bytes: int | None = None, + ) -> bool: # Small-head (<16) single-token decode takes either the Gluon kernel or # the padded asm persistent decode, selected by # VLLM_ROCM_AITER_MLA_ASM_PADDING and the arch (Gluon is gfx950 only). @@ -1497,10 +1535,12 @@ class AiterMLAHelper: mode = _aiter_mla_small_head_mode() if mode == "asm": return False - gluon_supported = _gluon_mla_decode_supported() - if mode == "gluon": - return gluon_supported - return m % num_heads == 0 and gluon_supported + if not _gluon_mla_decode_supported(): + return False + if mode != "gluon" and m % num_heads != 0: + return False + # Last, so the size warning only fires where Gluon would have run. + return _gluon_kv_cache_in_bounds(kv_cache_bytes) @staticmethod def use_gluon_verify( @@ -1509,6 +1549,7 @@ class AiterMLAHelper: kv_cache_dtype: str, dcp_world_size: int = 1, causal: bool = True, + kv_cache_bytes: int | None = None, ) -> bool: """Whether a small-head multi-token verify uses native Gluon MTP. @@ -1536,8 +1577,10 @@ class AiterMLAHelper: return False if not _gluon_mla_decode_supported(): return False - # Same arch and mode gating as use_gluon_decode. - return _aiter_mla_small_head_mode() != "asm" + # Same arch, mode and cache-size gating as use_gluon_decode. + if _aiter_mla_small_head_mode() == "asm": + return False + return _gluon_kv_cache_in_bounds(kv_cache_bytes) @staticmethod def dcp_local_verify_row_lens(