[ROCm][Spec Decode] Add Aiter MLA decode support non-causal draft block (#55966)
Signed-off-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com> Co-authored-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com> Co-authored-by: John Qin <zhengsheng1997@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>
This commit is contained in:
co-authored by
Poovaiah Palangappa
John Qin
Cursor
Hongxia Yang
parent
241e9391ed
commit
35dc273072
@@ -30,7 +30,10 @@ def reset_aiter_mla_support_cache(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
|
||||
def _install_fake_aiter_modules(
|
||||
monkeypatch: pytest.MonkeyPatch, *, supports_fp8: bool
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
supports_fp8: bool,
|
||||
supports_causal: bool = False,
|
||||
) -> None:
|
||||
aiter_mod: Any = types.ModuleType("aiter")
|
||||
mla_mod: Any = types.ModuleType("aiter.mla")
|
||||
@@ -65,6 +68,14 @@ def _install_fake_aiter_modules(
|
||||
|
||||
mla_decode_fwd = mla_decode_fwd_without_fp8
|
||||
|
||||
if supports_causal:
|
||||
inner = mla_decode_fwd
|
||||
|
||||
def mla_decode_fwd_with_causal(*args, causal=True, **kwargs):
|
||||
return inner(*args, **kwargs)
|
||||
|
||||
mla_decode_fwd = mla_decode_fwd_with_causal
|
||||
|
||||
mla_mod.mla_decode_fwd = mla_decode_fwd
|
||||
aiter_mod.mla = mla_mod
|
||||
|
||||
@@ -135,3 +146,107 @@ def test_aiter_mla_fp8_support_result_is_cached(monkeypatch):
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
assert signature_mock.call_count == 1
|
||||
|
||||
|
||||
def test_missing_causal_arg_fails_closed(monkeypatch):
|
||||
"""Old aiter wheels have no causal=; do not report that as a valid draft path."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=False)
|
||||
with pytest.raises(RuntimeError, match="causal-only"):
|
||||
rocm_aiter_ops.mla_decode_supports_non_causal()
|
||||
with pytest.raises(RuntimeError, match="causal-only"):
|
||||
AiterMLABackend.supports_non_causal()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_kernels", [False, True])
|
||||
def test_supports_non_causal_requires_arch_kernels(monkeypatch, has_kernels):
|
||||
"""The Python argument alone does not guarantee an arch has the kernel."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_non_causal_asm_kernels",
|
||||
lambda: has_kernels,
|
||||
)
|
||||
assert rocm_aiter_ops.mla_decode_supports_non_causal() is True
|
||||
assert AiterMLABackend.supports_non_causal() is has_kernels
|
||||
|
||||
|
||||
@pytest.mark.parametrize("causal", [True, False])
|
||||
def test_decode_fwd_forwards_causal_to_aiter(monkeypatch, causal):
|
||||
"""The decode op must pass the block mask through; omitting it is causal-only."""
|
||||
seen: list = []
|
||||
|
||||
def mla_decode_fwd(*args, causal=None, **kwargs):
|
||||
seen.append(causal)
|
||||
|
||||
mla_mod: Any = types.ModuleType("aiter.mla")
|
||||
mla_mod.mla_decode_fwd = mla_decode_fwd
|
||||
aiter_mod: Any = types.ModuleType("aiter")
|
||||
aiter_mod.mla = mla_mod
|
||||
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
|
||||
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
|
||||
|
||||
import torch
|
||||
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
|
||||
aiter_ops._rocm_aiter_mla_decode_fwd_impl(
|
||||
torch.zeros(2, 1, 8),
|
||||
torch.zeros(2, 1, 1, 8),
|
||||
torch.zeros(2, 1, 8),
|
||||
torch.zeros(3, dtype=torch.int32),
|
||||
1,
|
||||
causal=causal,
|
||||
)
|
||||
assert seen == [causal]
|
||||
|
||||
|
||||
def _aiter_mla_validate(
|
||||
monkeypatch,
|
||||
*,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
use_non_causal,
|
||||
):
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_non_causal_asm_kernels", lambda: True
|
||||
)
|
||||
return AiterMLABackend.validate_configuration(
|
||||
head_size=576,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
block_size=1,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=False,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(9, 5),
|
||||
attn_type=AttentionType.DECODER,
|
||||
use_non_causal=use_non_causal,
|
||||
)
|
||||
|
||||
|
||||
def test_non_causal_fp16_auto_cache_is_rejected(monkeypatch):
|
||||
"""Pinned AITER aborts fp16 Q; auto-select must fall through to Triton."""
|
||||
import torch
|
||||
|
||||
reasons = _aiter_mla_validate(
|
||||
monkeypatch,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="auto",
|
||||
use_non_causal=True,
|
||||
)
|
||||
assert any("non-causal fp16" in reason for reason in reasons)
|
||||
|
||||
@@ -176,3 +176,8 @@ def test_pad_unpad_round_trip_preserves_head_order(num_heads):
|
||||
|
||||
assert unpadded.shape == q.shape
|
||||
torch.testing.assert_close(unpadded, q)
|
||||
|
||||
|
||||
def test_a_non_causal_block_never_routes_to_gluon(gluon_available):
|
||||
"""A small-head block that normally uses Gluon must use ASM when non-causal."""
|
||||
assert not AiterMLAHelper.use_gluon_verify(12, 8, "auto", causal=False)
|
||||
|
||||
@@ -130,6 +130,7 @@ def _builder(
|
||||
_mla_q_dtype=torch.bfloat16,
|
||||
_mla_kv_dtype=torch.bfloat16,
|
||||
decode_attn_out_dtype=torch.bfloat16,
|
||||
_decode_causal=True,
|
||||
)
|
||||
# Bound to the stub rather than faked: the verify flatten's per-row view is
|
||||
# part of what _build_decode is being tested for.
|
||||
@@ -155,6 +156,25 @@ def test_backend_declares_uniform_batch_support():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("causal", [True, False])
|
||||
def test_build_hands_common_causal_flag_to_aiter_builder(monkeypatch, causal):
|
||||
seen = []
|
||||
|
||||
def fake_common_build(self, *args):
|
||||
seen.append(self._decode_causal)
|
||||
return SimpleNamespace(decode=None, prefill=None)
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla.MLACommonMetadataBuilder, "build", fake_common_build
|
||||
)
|
||||
builder = object.__new__(AiterMLAMetadataBuilder)
|
||||
builder._fp8_prefill_enabled = False
|
||||
|
||||
builder.build(0, SimpleNamespace(causal=causal))
|
||||
|
||||
assert seen == [causal]
|
||||
|
||||
|
||||
def test_dcp_verify_row_view_is_causal_per_row():
|
||||
"""Row t must cover this rank's share of positions [0, seq_len - qlen + t].
|
||||
|
||||
@@ -949,3 +969,81 @@ def test_persistent_metadata_gate_without_gluon_build(
|
||||
|
||||
assert metadata.has_persistent_metadata is expect_persistent
|
||||
assert get_mla_metadata_v1.called is expect_persistent
|
||||
|
||||
|
||||
def _build_non_causal(monkeypatch, *, num_heads, kv_cache_dtype, qlen, mtp_qlen):
|
||||
"""Drive _build_decode with causal=False and hand back the aiter mock."""
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
builder = _builder(
|
||||
mtp_decode_qlen=mtp_qlen,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
num_heads=num_heads,
|
||||
)
|
||||
builder._decode_causal = False
|
||||
metadata = AiterMLAMetadataBuilder._build_decode(
|
||||
builder,
|
||||
block_table_tensor=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
seq_lens_device=torch.tensor([7, 5], dtype=torch.int32),
|
||||
max_seq_len=7,
|
||||
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
num_decode_tokens=2 * qlen,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
return metadata, get_mla_metadata_v1
|
||||
|
||||
|
||||
def test_non_causal_build_hands_the_mask_to_aiter(monkeypatch):
|
||||
"""The schedule carries the mask, so is_causal must follow the block."""
|
||||
metadata, get_mla_metadata_v1 = _build_non_causal(
|
||||
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=8, mtp_qlen=8
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
# 6th positional arg of get_mla_metadata_v1 is is_causal.
|
||||
assert get_mla_metadata_v1.call_args.args[5] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [12, 16, 48])
|
||||
def test_a_two_token_fp8_block_is_refused_without_a_fold(monkeypatch, num_heads):
|
||||
"""16-head (and padded-to-16) qlen-2 fp8 has no non-causal kernel.
|
||||
|
||||
48/80/112 (and 33 padded to 48) fold to H16 while keeping Q2, so they
|
||||
must be refused the same way as native 16-head qlen 2.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="2-token"):
|
||||
_build_non_causal(
|
||||
monkeypatch,
|
||||
num_heads=num_heads,
|
||||
kv_cache_dtype="fp8",
|
||||
qlen=2,
|
||||
mtp_qlen=8,
|
||||
)
|
||||
|
||||
|
||||
def test_a_two_token_fp8_block_is_allowed_when_folded(monkeypatch):
|
||||
"""32 heads at qlen 2 folds onto the 16-head / 4-token kernel."""
|
||||
metadata, get_mla_metadata_v1 = _build_non_causal(
|
||||
monkeypatch,
|
||||
num_heads=32,
|
||||
kv_cache_dtype="fp8",
|
||||
qlen=2,
|
||||
mtp_qlen=8,
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
assert get_mla_metadata_v1.called
|
||||
|
||||
|
||||
def test_a_two_token_bf16_block_is_allowed(monkeypatch):
|
||||
"""The bf16 fold has no such hole, so the guard must not fire there."""
|
||||
metadata, _ = _build_non_causal(
|
||||
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
|
||||
Reference in New Issue
Block a user