[CI][ROCM] Add the Fusion E2E TP2 Quick group on MI355, and the AITER MLA fix it needs (#58369)

Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com>
Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <andreas.karatzas@protonmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
stefankoncarevic
2026-09-23 21:01:58 -05:00
committed by GitHub
co-authored by Andreas Karatzas Andreas Karatzas Codex
parent bcdacfc1ff
commit e9f167129c
7 changed files with 220 additions and 16 deletions
+41
View File
@@ -3773,6 +3773,47 @@ steps:
- pytest -v -s tests/distributed/test_rocm_quick_reduce.py
- pytest -v -s tests/distributed/test_quick_all_reduce.py
- label: ":amd: (MI355) Fusion E2E TP2 Quick" # TBD
timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_2
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/"
source_file_dependencies:
- csrc/quantization/
- vllm/compilation/
- vllm/distributed/
- vllm/model_executor/
- vllm/v1/attention/
- tests/compile/fusions_e2e/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
- >-
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
not +quant_fp8 and not +rms_norm and Llama-3 and
ROCM_AITER_UNIFIED_ATTN'
- >-
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
+quant_fp8 and +rms_norm and qwen3 and
ROCM_AITER_UNIFIED_ATTN'
- >-
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
not +quant_fp8 and not +rms_norm and DeepSeek-Coder and
ROCM_AITER_MLA'
#-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------#
- label: ":amd: (MI355) Entrypoints Integration (API Server) %N"
+41
View File
@@ -206,6 +206,47 @@ steps:
# Run all models and attn backends but only Inductor partition and native custom ops
- pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))"
- pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))"
mirror:
amd:
label: ":amd: (MI355) Fusion E2E TP2 Quick"
dind: false
device: mi355_2
timeout_in_minutes: 60
depends_on:
- image-build-amd
working_dir: /vllm-workspace/
source_file_dependencies:
- csrc/quantization/
- vllm/compilation/
- vllm/distributed/
- vllm/model_executor/
- vllm/v1/attention/
- tests/compile/fusions_e2e/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
- >-
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
not +quant_fp8 and not +rms_norm and Llama-3 and
ROCM_AITER_UNIFIED_ATTN'
- >-
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
+quant_fp8 and +rms_norm and qwen3 and
ROCM_AITER_UNIFIED_ATTN'
- >-
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
not +quant_fp8 and not +rms_norm and DeepSeek-Coder and
ROCM_AITER_MLA'
- label: ":nvidia: (H100) Fusion E2E TP2 AR-RMS Config Sweep"
key: fusion-e2e-tp2-ar-rms-config-sweep-h100
+9
View File
@@ -44,6 +44,15 @@ ROCM_AITER_UNIFIED_ATTN = pytest.param(
),
)
ROCM_AITER_MLA_ATTN = pytest.param(
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_AITER_MLA),
id="ROCM_AITER_MLA",
marks=pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="ROCM_AITER_MLA only for AMD when AITER is installed",
),
)
FLASHINFER_MLA_ATTN = pytest.param(
AttentionBackendCase(backend=AttentionBackendEnum.FLASHINFER_MLA),
id="FLASHINFER_MLA",
+57 -8
View File
@@ -19,6 +19,7 @@ from .common import (
from .models import (
FLASHINFER_ATTN,
FLASHINFER_MLA_ATTN,
ROCM_AITER_MLA_ATTN,
ROCM_AITER_UNIFIED_ATTN,
ROCM_ATTN,
TRITON_ATTN,
@@ -40,25 +41,35 @@ pytestmark = pytest.mark.skipif(
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet supported
# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet
# supported. ROCm carries only the payloads its two backends support.
TP2_FP8_MODELS = (
[
llama3_8b_fp8,
llama4_scout_fp8,
qwen3_a3b_fp8,
deepseek_coder_v2_lite_fp8,
deepseek_v3_fp8,
],
]
if current_platform.is_cuda()
else [llama3_8b_fp8, qwen3_a3b_fp8, deepseek_coder_v2_lite_fp8]
)
TP2_FP8_ATTN_BACKENDS = (
[TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN]
if current_platform.is_cuda()
else [ROCM_AITER_UNIFIED_ATTN, ROCM_AITER_MLA_ATTN]
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"attn_backend", [TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN]
"model_name, matches_fn, model_kwargs, hf_overrides", TP2_FP8_MODELS
)
@pytest.mark.parametrize("attn_backend", TP2_FP8_ATTN_BACKENDS)
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
def test_tp2_ar_rms_fp8_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
@@ -69,10 +80,27 @@ def test_tp2_ar_rms_fp8_fusions(
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
monkeypatch,
):
matches = matches_fn(n_layers)
block_fp8 = "qwen" in model_name.lower() or "deepseek" in model_name.lower()
model_name_lower = model_name.lower()
# DeepSeek-Coder on AITER MLA is the one static-FP8 payload here, validated
# against the native QuantFP8 pattern; every other DeepSeek stays group
# quantized as on CUDA.
rocm_static_fp8_mla = (
current_platform.is_rocm()
and "deepseek-coder" in model_name_lower
and attn_backend.backend.name == "ROCM_AITER_MLA"
)
block_fp8 = "qwen" in model_name_lower or "deepseek" in model_name_lower
if rocm_static_fp8_mla:
block_fp8 = False
if custom_ops != "-quant_fp8,-rms_norm":
pytest.skip("ROCm static-FP8 MLA parity uses native quant and RMSNorm")
# AITER's MoE GEMM rejects this reduced dummy model shape; keep AITER
# for MLA and the all-reduce, and let the MoE run on Triton.
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "0")
if block_fp8 and "-quant_fp8" in custom_ops:
# This is why config forces +quant_fp8 by default
pytest.skip("native QuantFP8 matching not supported for group quant")
@@ -83,6 +111,13 @@ def test_tp2_ar_rms_fp8_fusions(
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
model_kwargs["disable_custom_all_reduce"] = False
if current_platform.is_rocm() and "llama-3" in model_name_lower:
model_kwargs.update(
gpu_memory_utilization=0.01,
kv_cache_memory_bytes=64 * 1024**2,
max_num_batched_tokens=1024,
max_num_seqs=32,
)
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
@@ -103,6 +138,19 @@ def test_tp2_ar_rms_fp8_fusions(
"ar_rms_fusion",
]
use_aiter = current_platform.is_rocm()
if use_aiter:
matches = matches._replace(aiter_ar_rms_fusion=matches.ar_rms_fusion)
# The AITER all-reduce pass registers the AR+RMS(+group-quant) patterns
# before its AR+RMS-only ones, so the standalone AITER RMS-quant pass
# does not own the post-AR sites here. ROCm's contract is native
# QK-norm+RoPE and attention-output quant fusion plus AITER AR+RMS.
matches_check = [
"norm_rope_fusion",
"attn_quant_fusion",
"aiter_ar_rms_fusion",
]
run_e2e_fusion_test(
model_name,
matches,
@@ -111,6 +159,7 @@ def test_tp2_ar_rms_fp8_fusions(
compilation_config,
matches_check,
tp_size=2,
use_aiter=use_aiter,
)
@@ -151,6 +151,25 @@ def test_decode_and_verify_are_disjoint(kv_cache_dtype, num_heads):
)
@pytest.mark.parametrize("kv_cache_dtype", UNQUANTIZED_DTYPES)
def test_oversized_kv_cache_refuses_gluon(gluon_available, kv_cache_dtype):
"""Past 2 GiB per layer Gluon drops its KV bounds mask and reads out of range.
``None`` is the profiling case, before the cache is sized: nothing to
refuse yet, so the usual route stands.
"""
bound = rocm_aiter_mla._GLUON_MAX_KV_CACHE_BYTES
for kv_cache_bytes in (None, bound):
assert AiterMLAHelper.use_gluon_decode(8, 1, kv_cache_dtype, kv_cache_bytes)
assert AiterMLAHelper.use_gluon_verify(
8, 8, kv_cache_dtype, kv_cache_bytes=kv_cache_bytes
)
assert not AiterMLAHelper.use_gluon_decode(8, 1, kv_cache_dtype, bound + 1)
assert not AiterMLAHelper.use_gluon_verify(
8, 8, kv_cache_dtype, kv_cache_bytes=bound + 1
)
@pytest.mark.parametrize("num_heads", [1, 2, 3, 5, 6, 7, 8, 9, 12, 15])
def test_padded_query_is_contiguous(num_heads):
"""asm_mla.cu:805 requires Q.is_contiguous().
@@ -107,6 +107,8 @@ def _builder(
_dcp_verify_buffers=None,
_graph_seq_lens=None,
_kv_cache_dtype_str=kv_cache_dtype,
# Unsized as during profiling, so the Gluon KV bound stays out of the way.
_kv_cache_bytes=None,
paged_kv_last_page_len=torch.ones(max_decode_rows, dtype=torch.int32),
paged_kv_indices=torch.empty(1024, dtype=torch.int32),
paged_kv_indptr=torch.empty(max_decode_rows + 1, dtype=torch.int32),
@@ -585,7 +587,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
get_num_attention_heads=lambda parallel_config: num_heads,
),
scheduler_config=SimpleNamespace(max_num_seqs=2),
cache_config=SimpleNamespace(cache_dtype="fp8_e4m3"),
cache_config=SimpleNamespace(cache_dtype="fp8_e4m3", num_gpu_blocks=None),
compilation_config=SimpleNamespace(
cudagraph_mode=SimpleNamespace(has_full_cudagraphs=lambda: False),
# Empty: the per-layer head-count probe finds no attention layer and
@@ -178,6 +178,30 @@ def _gluon_mla_decode_supported() -> bool:
return on_gfx950()
# Past 2 GiB, mla_gluon swaps its masked buffer_load for an unmasked
# global_load that reads outside the cache and aborts the process. Mirrors the
# kernel's own within_2gb bound, so the fallback engages exactly where it does.
_GLUON_MAX_KV_CACHE_BYTES = 1 << 31
def _gluon_kv_cache_in_bounds(kv_cache_bytes: int | None) -> bool:
"""Whether this layer's KV cache keeps Gluon on its bounds-checked path.
The cache is one flat tensor per layer, so the bound disqualifies the layer
outright; no batch shape brings an oversized cache back in range. ``None``
is the not-yet-sized case during profiling, before any kernel runs.
"""
if kv_cache_bytes is None or kv_cache_bytes <= _GLUON_MAX_KV_CACHE_BYTES:
return True
logger.warning_once(
"KV cache is %.1f GiB per layer, past the 2 GiB where the Gluon MLA "
"kernel drops its KV bounds mask; using the padded ASM decode instead. "
"Lower --gpu-memory-utilization or --max-model-len to get Gluon back.",
kv_cache_bytes / (1 << 30),
)
return False
@functools.lru_cache(maxsize=1)
def _get_segmented_mla_decode():
"""Load AITER's segmented MLA decode with unreduced partial output."""
@@ -564,6 +588,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# _build_decode needs the cache dtype to pick the decode kernel; keep
# the normalized string instead of dropping it at the end of __init__.
self._kv_cache_dtype_str = kv_cache_dtype_str
# Sized per layer, matching the flat KV tensor the kernel is handed.
num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks
self._kv_cache_bytes = (
None
if num_gpu_blocks is None
else num_gpu_blocks * kv_cache_spec.page_size_bytes
)
# MLAAttention quantizes decode Q to FP8 before calling this backend
# whenever the KV cache is FP8 and supports_quant_query_input is true.
q_dtype = (
@@ -1065,6 +1096,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
self._decode_num_heads,
int(max_qo_len),
self._kv_cache_dtype_str,
self._kv_cache_bytes,
)
use_gluon_verify = AiterMLAHelper.use_gluon_verify(
self._decode_num_heads,
@@ -1072,6 +1104,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
self._kv_cache_dtype_str,
self.dcp_world_size,
causal,
self._kv_cache_bytes,
)
use_segmented_dcp_verify = (
self._supports_segmented_dcp_verify and max_qo_len > 1
@@ -1481,7 +1514,12 @@ class AiterMLAHelper:
return AiterMLAHelper._get_mla_unpadded_heads(num_heads, lse)
@staticmethod
def use_gluon_decode(num_heads: int, max_qo_len: int, kv_cache_dtype: str) -> bool:
def use_gluon_decode(
num_heads: int,
max_qo_len: int,
kv_cache_dtype: str,
kv_cache_bytes: int | None = None,
) -> bool:
# Small-head (<16) single-token decode takes either the Gluon kernel or
# the padded asm persistent decode, selected by
# VLLM_ROCM_AITER_MLA_ASM_PADDING and the arch (Gluon is gfx950 only).
@@ -1497,10 +1535,12 @@ class AiterMLAHelper:
mode = _aiter_mla_small_head_mode()
if mode == "asm":
return False
gluon_supported = _gluon_mla_decode_supported()
if mode == "gluon":
return gluon_supported
return m % num_heads == 0 and gluon_supported
if not _gluon_mla_decode_supported():
return False
if mode != "gluon" and m % num_heads != 0:
return False
# Last, so the size warning only fires where Gluon would have run.
return _gluon_kv_cache_in_bounds(kv_cache_bytes)
@staticmethod
def use_gluon_verify(
@@ -1509,6 +1549,7 @@ class AiterMLAHelper:
kv_cache_dtype: str,
dcp_world_size: int = 1,
causal: bool = True,
kv_cache_bytes: int | None = None,
) -> bool:
"""Whether a small-head multi-token verify uses native Gluon MTP.
@@ -1536,8 +1577,10 @@ class AiterMLAHelper:
return False
if not _gluon_mla_decode_supported():
return False
# Same arch and mode gating as use_gluon_decode.
return _aiter_mla_small_head_mode() != "asm"
# Same arch, mode and cache-size gating as use_gluon_decode.
if _aiter_mla_small_head_mode() == "asm":
return False
return _gluon_kv_cache_in_bounds(kv_cache_bytes)
@staticmethod
def dcp_local_verify_row_lens(