[CI][ROCM] Add the Fusion E2E TP2 Quick group on MI355, and the AITER MLA fix it needs (#58369)
Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com> Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Andreas Karatzas <andreas.karatzas@protonmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Andreas Karatzas
Andreas Karatzas
Codex
parent
bcdacfc1ff
commit
e9f167129c
@@ -3773,6 +3773,47 @@ steps:
|
||||
- pytest -v -s tests/distributed/test_rocm_quick_reduce.py
|
||||
- pytest -v -s tests/distributed/test_quick_all_reduce.py
|
||||
|
||||
- label: ":amd: (MI355) Fusion E2E TP2 Quick" # TBD
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_2
|
||||
num_gpus: 2
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/"
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/
|
||||
- vllm/compilation/
|
||||
- vllm/distributed/
|
||||
- vllm/model_executor/
|
||||
- vllm/v1/attention/
|
||||
- tests/compile/fusions_e2e/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
- >-
|
||||
VLLM_ROCM_USE_AITER=1
|
||||
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
|
||||
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
|
||||
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
|
||||
not +quant_fp8 and not +rms_norm and Llama-3 and
|
||||
ROCM_AITER_UNIFIED_ATTN'
|
||||
- >-
|
||||
VLLM_ROCM_USE_AITER=1
|
||||
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
|
||||
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
|
||||
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
|
||||
+quant_fp8 and +rms_norm and qwen3 and
|
||||
ROCM_AITER_UNIFIED_ATTN'
|
||||
- >-
|
||||
VLLM_ROCM_USE_AITER=1
|
||||
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
|
||||
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
|
||||
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
|
||||
not +quant_fp8 and not +rms_norm and DeepSeek-Coder and
|
||||
ROCM_AITER_MLA'
|
||||
|
||||
#-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------#
|
||||
|
||||
- label: ":amd: (MI355) Entrypoints Integration (API Server) %N"
|
||||
|
||||
@@ -206,6 +206,47 @@ steps:
|
||||
# Run all models and attn backends but only Inductor partition and native custom ops
|
||||
- pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))"
|
||||
- pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))"
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Fusion E2E TP2 Quick"
|
||||
dind: false
|
||||
device: mi355_2
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
working_dir: /vllm-workspace/
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/
|
||||
- vllm/compilation/
|
||||
- vllm/distributed/
|
||||
- vllm/model_executor/
|
||||
- vllm/v1/attention/
|
||||
- tests/compile/fusions_e2e/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
- >-
|
||||
VLLM_ROCM_USE_AITER=1
|
||||
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
|
||||
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
|
||||
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
|
||||
not +quant_fp8 and not +rms_norm and Llama-3 and
|
||||
ROCM_AITER_UNIFIED_ATTN'
|
||||
- >-
|
||||
VLLM_ROCM_USE_AITER=1
|
||||
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
|
||||
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
|
||||
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
|
||||
+quant_fp8 and +rms_norm and qwen3 and
|
||||
ROCM_AITER_UNIFIED_ATTN'
|
||||
- >-
|
||||
VLLM_ROCM_USE_AITER=1
|
||||
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
|
||||
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
|
||||
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
|
||||
not +quant_fp8 and not +rms_norm and DeepSeek-Coder and
|
||||
ROCM_AITER_MLA'
|
||||
|
||||
- label: ":nvidia: (H100) Fusion E2E TP2 AR-RMS Config Sweep"
|
||||
key: fusion-e2e-tp2-ar-rms-config-sweep-h100
|
||||
|
||||
@@ -44,6 +44,15 @@ ROCM_AITER_UNIFIED_ATTN = pytest.param(
|
||||
),
|
||||
)
|
||||
|
||||
ROCM_AITER_MLA_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_AITER_MLA),
|
||||
id="ROCM_AITER_MLA",
|
||||
marks=pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="ROCM_AITER_MLA only for AMD when AITER is installed",
|
||||
),
|
||||
)
|
||||
|
||||
FLASHINFER_MLA_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.FLASHINFER_MLA),
|
||||
id="FLASHINFER_MLA",
|
||||
|
||||
@@ -19,6 +19,7 @@ from .common import (
|
||||
from .models import (
|
||||
FLASHINFER_ATTN,
|
||||
FLASHINFER_MLA_ATTN,
|
||||
ROCM_AITER_MLA_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
ROCM_ATTN,
|
||||
TRITON_ATTN,
|
||||
@@ -40,25 +41,35 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet supported
|
||||
# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet
|
||||
# supported. ROCm carries only the payloads its two backends support.
|
||||
TP2_FP8_MODELS = (
|
||||
[
|
||||
llama3_8b_fp8,
|
||||
llama4_scout_fp8,
|
||||
qwen3_a3b_fp8,
|
||||
deepseek_coder_v2_lite_fp8,
|
||||
deepseek_v3_fp8,
|
||||
],
|
||||
]
|
||||
if current_platform.is_cuda()
|
||||
else [llama3_8b_fp8, qwen3_a3b_fp8, deepseek_coder_v2_lite_fp8]
|
||||
)
|
||||
|
||||
TP2_FP8_ATTN_BACKENDS = (
|
||||
[TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN]
|
||||
if current_platform.is_cuda()
|
||||
else [ROCM_AITER_UNIFIED_ATTN, ROCM_AITER_MLA_ATTN]
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend", [TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN]
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides", TP2_FP8_MODELS
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", TP2_FP8_ATTN_BACKENDS)
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
def test_tp2_ar_rms_fp8_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
@@ -69,10 +80,27 @@ def test_tp2_ar_rms_fp8_fusions(
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
monkeypatch,
|
||||
):
|
||||
matches = matches_fn(n_layers)
|
||||
|
||||
block_fp8 = "qwen" in model_name.lower() or "deepseek" in model_name.lower()
|
||||
model_name_lower = model_name.lower()
|
||||
# DeepSeek-Coder on AITER MLA is the one static-FP8 payload here, validated
|
||||
# against the native QuantFP8 pattern; every other DeepSeek stays group
|
||||
# quantized as on CUDA.
|
||||
rocm_static_fp8_mla = (
|
||||
current_platform.is_rocm()
|
||||
and "deepseek-coder" in model_name_lower
|
||||
and attn_backend.backend.name == "ROCM_AITER_MLA"
|
||||
)
|
||||
block_fp8 = "qwen" in model_name_lower or "deepseek" in model_name_lower
|
||||
if rocm_static_fp8_mla:
|
||||
block_fp8 = False
|
||||
if custom_ops != "-quant_fp8,-rms_norm":
|
||||
pytest.skip("ROCm static-FP8 MLA parity uses native quant and RMSNorm")
|
||||
# AITER's MoE GEMM rejects this reduced dummy model shape; keep AITER
|
||||
# for MLA and the all-reduce, and let the MoE run on Triton.
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "0")
|
||||
if block_fp8 and "-quant_fp8" in custom_ops:
|
||||
# This is why config forces +quant_fp8 by default
|
||||
pytest.skip("native QuantFP8 matching not supported for group quant")
|
||||
@@ -83,6 +111,13 @@ def test_tp2_ar_rms_fp8_fusions(
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
model_kwargs["disable_custom_all_reduce"] = False
|
||||
if current_platform.is_rocm() and "llama-3" in model_name_lower:
|
||||
model_kwargs.update(
|
||||
gpu_memory_utilization=0.01,
|
||||
kv_cache_memory_bytes=64 * 1024**2,
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_seqs=32,
|
||||
)
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
@@ -103,6 +138,19 @@ def test_tp2_ar_rms_fp8_fusions(
|
||||
"ar_rms_fusion",
|
||||
]
|
||||
|
||||
use_aiter = current_platform.is_rocm()
|
||||
if use_aiter:
|
||||
matches = matches._replace(aiter_ar_rms_fusion=matches.ar_rms_fusion)
|
||||
# The AITER all-reduce pass registers the AR+RMS(+group-quant) patterns
|
||||
# before its AR+RMS-only ones, so the standalone AITER RMS-quant pass
|
||||
# does not own the post-AR sites here. ROCm's contract is native
|
||||
# QK-norm+RoPE and attention-output quant fusion plus AITER AR+RMS.
|
||||
matches_check = [
|
||||
"norm_rope_fusion",
|
||||
"attn_quant_fusion",
|
||||
"aiter_ar_rms_fusion",
|
||||
]
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
@@ -111,6 +159,7 @@ def test_tp2_ar_rms_fp8_fusions(
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
use_aiter=use_aiter,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -151,6 +151,25 @@ def test_decode_and_verify_are_disjoint(kv_cache_dtype, num_heads):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", UNQUANTIZED_DTYPES)
|
||||
def test_oversized_kv_cache_refuses_gluon(gluon_available, kv_cache_dtype):
|
||||
"""Past 2 GiB per layer Gluon drops its KV bounds mask and reads out of range.
|
||||
|
||||
``None`` is the profiling case, before the cache is sized: nothing to
|
||||
refuse yet, so the usual route stands.
|
||||
"""
|
||||
bound = rocm_aiter_mla._GLUON_MAX_KV_CACHE_BYTES
|
||||
for kv_cache_bytes in (None, bound):
|
||||
assert AiterMLAHelper.use_gluon_decode(8, 1, kv_cache_dtype, kv_cache_bytes)
|
||||
assert AiterMLAHelper.use_gluon_verify(
|
||||
8, 8, kv_cache_dtype, kv_cache_bytes=kv_cache_bytes
|
||||
)
|
||||
assert not AiterMLAHelper.use_gluon_decode(8, 1, kv_cache_dtype, bound + 1)
|
||||
assert not AiterMLAHelper.use_gluon_verify(
|
||||
8, 8, kv_cache_dtype, kv_cache_bytes=bound + 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [1, 2, 3, 5, 6, 7, 8, 9, 12, 15])
|
||||
def test_padded_query_is_contiguous(num_heads):
|
||||
"""asm_mla.cu:805 requires Q.is_contiguous().
|
||||
|
||||
@@ -107,6 +107,8 @@ def _builder(
|
||||
_dcp_verify_buffers=None,
|
||||
_graph_seq_lens=None,
|
||||
_kv_cache_dtype_str=kv_cache_dtype,
|
||||
# Unsized as during profiling, so the Gluon KV bound stays out of the way.
|
||||
_kv_cache_bytes=None,
|
||||
paged_kv_last_page_len=torch.ones(max_decode_rows, dtype=torch.int32),
|
||||
paged_kv_indices=torch.empty(1024, dtype=torch.int32),
|
||||
paged_kv_indptr=torch.empty(max_decode_rows + 1, dtype=torch.int32),
|
||||
@@ -585,7 +587,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
get_num_attention_heads=lambda parallel_config: num_heads,
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=2),
|
||||
cache_config=SimpleNamespace(cache_dtype="fp8_e4m3"),
|
||||
cache_config=SimpleNamespace(cache_dtype="fp8_e4m3", num_gpu_blocks=None),
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=SimpleNamespace(has_full_cudagraphs=lambda: False),
|
||||
# Empty: the per-layer head-count probe finds no attention layer and
|
||||
|
||||
@@ -178,6 +178,30 @@ def _gluon_mla_decode_supported() -> bool:
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
# Past 2 GiB, mla_gluon swaps its masked buffer_load for an unmasked
|
||||
# global_load that reads outside the cache and aborts the process. Mirrors the
|
||||
# kernel's own within_2gb bound, so the fallback engages exactly where it does.
|
||||
_GLUON_MAX_KV_CACHE_BYTES = 1 << 31
|
||||
|
||||
|
||||
def _gluon_kv_cache_in_bounds(kv_cache_bytes: int | None) -> bool:
|
||||
"""Whether this layer's KV cache keeps Gluon on its bounds-checked path.
|
||||
|
||||
The cache is one flat tensor per layer, so the bound disqualifies the layer
|
||||
outright; no batch shape brings an oversized cache back in range. ``None``
|
||||
is the not-yet-sized case during profiling, before any kernel runs.
|
||||
"""
|
||||
if kv_cache_bytes is None or kv_cache_bytes <= _GLUON_MAX_KV_CACHE_BYTES:
|
||||
return True
|
||||
logger.warning_once(
|
||||
"KV cache is %.1f GiB per layer, past the 2 GiB where the Gluon MLA "
|
||||
"kernel drops its KV bounds mask; using the padded ASM decode instead. "
|
||||
"Lower --gpu-memory-utilization or --max-model-len to get Gluon back.",
|
||||
kv_cache_bytes / (1 << 30),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _get_segmented_mla_decode():
|
||||
"""Load AITER's segmented MLA decode with unreduced partial output."""
|
||||
@@ -564,6 +588,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
# _build_decode needs the cache dtype to pick the decode kernel; keep
|
||||
# the normalized string instead of dropping it at the end of __init__.
|
||||
self._kv_cache_dtype_str = kv_cache_dtype_str
|
||||
# Sized per layer, matching the flat KV tensor the kernel is handed.
|
||||
num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks
|
||||
self._kv_cache_bytes = (
|
||||
None
|
||||
if num_gpu_blocks is None
|
||||
else num_gpu_blocks * kv_cache_spec.page_size_bytes
|
||||
)
|
||||
# MLAAttention quantizes decode Q to FP8 before calling this backend
|
||||
# whenever the KV cache is FP8 and supports_quant_query_input is true.
|
||||
q_dtype = (
|
||||
@@ -1065,6 +1096,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
self._decode_num_heads,
|
||||
int(max_qo_len),
|
||||
self._kv_cache_dtype_str,
|
||||
self._kv_cache_bytes,
|
||||
)
|
||||
use_gluon_verify = AiterMLAHelper.use_gluon_verify(
|
||||
self._decode_num_heads,
|
||||
@@ -1072,6 +1104,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
self._kv_cache_dtype_str,
|
||||
self.dcp_world_size,
|
||||
causal,
|
||||
self._kv_cache_bytes,
|
||||
)
|
||||
use_segmented_dcp_verify = (
|
||||
self._supports_segmented_dcp_verify and max_qo_len > 1
|
||||
@@ -1481,7 +1514,12 @@ class AiterMLAHelper:
|
||||
return AiterMLAHelper._get_mla_unpadded_heads(num_heads, lse)
|
||||
|
||||
@staticmethod
|
||||
def use_gluon_decode(num_heads: int, max_qo_len: int, kv_cache_dtype: str) -> bool:
|
||||
def use_gluon_decode(
|
||||
num_heads: int,
|
||||
max_qo_len: int,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_bytes: int | None = None,
|
||||
) -> bool:
|
||||
# Small-head (<16) single-token decode takes either the Gluon kernel or
|
||||
# the padded asm persistent decode, selected by
|
||||
# VLLM_ROCM_AITER_MLA_ASM_PADDING and the arch (Gluon is gfx950 only).
|
||||
@@ -1497,10 +1535,12 @@ class AiterMLAHelper:
|
||||
mode = _aiter_mla_small_head_mode()
|
||||
if mode == "asm":
|
||||
return False
|
||||
gluon_supported = _gluon_mla_decode_supported()
|
||||
if mode == "gluon":
|
||||
return gluon_supported
|
||||
return m % num_heads == 0 and gluon_supported
|
||||
if not _gluon_mla_decode_supported():
|
||||
return False
|
||||
if mode != "gluon" and m % num_heads != 0:
|
||||
return False
|
||||
# Last, so the size warning only fires where Gluon would have run.
|
||||
return _gluon_kv_cache_in_bounds(kv_cache_bytes)
|
||||
|
||||
@staticmethod
|
||||
def use_gluon_verify(
|
||||
@@ -1509,6 +1549,7 @@ class AiterMLAHelper:
|
||||
kv_cache_dtype: str,
|
||||
dcp_world_size: int = 1,
|
||||
causal: bool = True,
|
||||
kv_cache_bytes: int | None = None,
|
||||
) -> bool:
|
||||
"""Whether a small-head multi-token verify uses native Gluon MTP.
|
||||
|
||||
@@ -1536,8 +1577,10 @@ class AiterMLAHelper:
|
||||
return False
|
||||
if not _gluon_mla_decode_supported():
|
||||
return False
|
||||
# Same arch and mode gating as use_gluon_decode.
|
||||
return _aiter_mla_small_head_mode() != "asm"
|
||||
# Same arch, mode and cache-size gating as use_gluon_decode.
|
||||
if _aiter_mla_small_head_mode() == "asm":
|
||||
return False
|
||||
return _gluon_kv_cache_in_bounds(kv_cache_bytes)
|
||||
|
||||
@staticmethod
|
||||
def dcp_local_verify_row_lens(
|
||||
|
||||
Reference in New Issue
Block a user