[Perf] [Feat] [ROCm] Add densemha support to ROCm AITER Sparse MLA (#49263)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
TJian
2026-10-12 15:50:28 +00:00
committed by GitHub
parent a686c8e1b7
commit 50b21a2667
7 changed files with 984 additions and 142 deletions
+7 -3
View File
@@ -2632,8 +2632,11 @@ steps:
- vllm/_aiter_ops.py
- vllm/envs.py
- vllm/platforms/rocm.py
- vllm/models/deepseek_v32/amd/rocm.py
commands:
- pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- pytest -v -s v1/attention --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":amd: (MI300) Metrics, Tracing" # TBD
timeout_in_minutes: 30
@@ -5642,10 +5645,11 @@ steps:
- vllm/_aiter_ops.py
- vllm/envs.py
- vllm/platforms/rocm.py
- vllm/models/deepseek_v32/amd/rocm.py
optional: true
commands:
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":amd: (MI355 DPX) CUDAGraph"
timeout_in_minutes: 55
+7 -2
View File
@@ -30,6 +30,10 @@ steps:
- vllm/_aiter_ops.py
- vllm/envs.py
- vllm/platforms/rocm.py
- vllm/models/deepseek_v32/amd/rocm.py
commands:
- pytest -v -s v1/attention --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":nvidia: (B200) V1 Attention Shard %N"
key: v1-attention-b200
@@ -59,9 +63,10 @@ steps:
- vllm/_aiter_ops.py
- vllm/envs.py
- vllm/platforms/rocm.py
- vllm/models/deepseek_v32/amd/rocm.py
commands:
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":nvidia: (DGX) Spark FlashInfer NVFP4 XQA"
key: flashinfer-nvfp4-xqa-dgx-spark
@@ -33,6 +33,101 @@ class _FakeAiter(ModuleType):
get_mla_metadata_v1: Mock
@pytest.mark.parametrize("block_size", [16, 64, 128])
@pytest.mark.parametrize("topk_tokens", [128, 256])
def test_rocm_index_conversion_matrix(block_size, topk_tokens):
"""Convert valid, masked, and out-of-range sparse indices exactly."""
device = torch.device("cuda")
num_tokens = 4
max_blocks = 4
req_ids = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device=device)
block_table = torch.tensor(
[[3, 5, 7, 9], [2, 4, 6, 8]], dtype=torch.int32, device=device
)
token_indices = torch.arange(
num_tokens * topk_tokens, dtype=torch.int32, device=device
).view(num_tokens, topk_tokens)
token_indices %= max_blocks * block_size
token_indices[0, 0] = -1
token_indices[1, 1] = max_blocks * block_size
cu_seqlens = torch.arange(
0,
(num_tokens + 1) * topk_tokens,
topk_tokens,
dtype=torch.int32,
device=device,
)
output = torch.full(
(num_tokens * topk_tokens,), -99, dtype=torch.int32, device=device
)
sparse_mod.triton_convert_req_index_to_global_index(
req_ids,
block_table,
token_indices,
cu_seqlens,
output,
BLOCK_STRIDE_ROWS=block_size,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=topk_tokens,
)
block_ids = token_indices // block_size
valid = (token_indices >= 0) & (block_ids < max_blocks)
safe_blocks = block_ids.clamp(0, max_blocks - 1)
bases = block_table[req_ids[:, None].long(), safe_blocks.long()]
expected = torch.where(valid, bases * block_size + token_indices % block_size, 0)
torch.testing.assert_close(output.view_as(expected), expected, rtol=0, atol=0)
def test_rocm_index_conversion_rejects_mismatched_token_extents():
"""Reject full-batch request IDs paired with decode-subset indices."""
with pytest.raises(AssertionError, match="must cover the same tokens"):
sparse_mod.triton_convert_req_index_to_global_index(
torch.zeros(3, dtype=torch.int32),
torch.zeros(1, 4, dtype=torch.int32),
torch.zeros(2, 128, dtype=torch.int32),
torch.arange(3, dtype=torch.int32),
torch.zeros(256, dtype=torch.int32),
BLOCK_STRIDE_ROWS=64,
NUM_TOPK_TOKENS=128,
)
def test_rocm_index_conversion_respects_packed_valid_counts():
"""ROCm's indptr contract packs only each token's valid top-k prefix."""
device = torch.device("cuda")
topk_tokens = 128
valid_counts = torch.tensor([1, 17, 128], dtype=torch.int32, device=device)
cu_seqlens = torch.zeros(4, dtype=torch.int32, device=device)
torch.cumsum(valid_counts, dim=0, out=cu_seqlens[1:])
req_ids = torch.tensor([0, 0, 0], dtype=torch.int32, device=device)
block_table = torch.tensor([[2, 3, 4]], dtype=torch.int32, device=device)
token_indices = torch.arange(
3 * topk_tokens, dtype=torch.int32, device=device
).view(3, topk_tokens)
token_indices %= 3 * 64
output = torch.full((int(cu_seqlens[-1]),), -99, dtype=torch.int32, device=device)
sparse_mod.triton_convert_req_index_to_global_index(
req_ids,
block_table,
token_indices,
cu_seqlens,
output,
BLOCK_STRIDE_ROWS=64,
BLOCK_SIZE=64,
NUM_TOPK_TOKENS=topk_tokens,
)
expected_parts = []
for token_id, count in enumerate(valid_counts.tolist()):
local = token_indices[token_id, :count]
expected_parts.append(block_table[0, (local // 64).long()] * 64 + local % 64)
expected = torch.cat(expected_parts)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def _make_builder():
builder = object.__new__(sparse_mod.ROCMAiterMLASparseMetadataBuilder)
max_num_batched_tokens = 8
@@ -44,6 +139,18 @@ def _make_builder():
builder.kv_cache_dtype = "fp8"
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
builder.topk_tokens = topk_tokens
# SparseMLACommonMetadataBuilder state consumed by its build() method.
builder.cp_kv_cache_interleave_size = 1
builder.use_pcp = False
builder._prefill_backend = None
def cpu_token_to_req(metadata):
return torch.repeat_interleave(
torch.arange(metadata.num_reqs, dtype=torch.int32),
metadata.query_start_loc_cpu.diff(),
)
builder._build_req_id_per_token = cpu_token_to_req # type: ignore[method-assign]
builder.req_id_per_token_buffer = torch.zeros(
max_num_batched_tokens, dtype=torch.int32, device="cpu"
)
@@ -62,6 +169,8 @@ def _make_builder():
builder._use_persistent_metadata = True
builder._num_attention_heads = 16
builder._num_compute_units = current_platform.num_compute_units()
builder._mla_q_dtype = torch.bfloat16
builder._mla_kv_dtype = torch.bfloat16
builder._mla_work_meta_data = torch.empty(1, dtype=torch.int32, device="cpu")
builder._mla_work_indptr = torch.empty(1, dtype=torch.int32, device="cpu")
builder._mla_work_info_set = torch.empty(1, dtype=torch.int32, device="cpu")
@@ -216,6 +325,8 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
assert events == ["metadata", "sync"]
assert fake_get_mla_metadata_v1_mock.call_count == 1
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["max_split_per_batch"] == 1
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_q"] == torch.bfloat16
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_kv"] == torch.bfloat16
events.clear()
@@ -223,3 +334,395 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
assert events == []
assert fake_get_mla_metadata_v1_mock.call_count == 1
def test_dense_prefill_builds_sparse_metadata_for_decode_subset(monkeypatch):
"""Dense prefill must exclude prefill tokens from AITER sparse metadata.
A mixed batch is routed as sparse MQA for its leading decode requests and
dense MHA for its trailing short prefill requests. The AITER metadata
buffers therefore must describe only the decode subset; including prefill
tokens would mismatch the query passed to ``forward_mqa``.
"""
builder = _make_builder()
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
)
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
seq_lens = torch.tensor([16, 4], dtype=torch.int32)
common_metadata = SimpleNamespace(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
max_query_len=4,
max_seq_len=16,
)
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
num_reqs=2,
max_query_len=4,
max_seq_len=16,
num_actual_tokens=5,
query_start_loc=query_start_loc,
slot_mapping=torch.arange(5),
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
req_id_per_token=torch.tensor([0, 1, 1, 1, 1], dtype=torch.int32),
topk_tokens=4,
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
seq_lens=seq_lens,
prefill_max_seq_len=4,
prefill=SimpleNamespace(use_dense_mha=True),
)
monkeypatch.setattr(
sparse_mod.SparseMLACommonMetadataBuilder,
"build",
lambda *args, **kwargs: metadata,
)
sparse_call: dict[str, torch.Tensor] = {}
def fake_generate_sparse_seqlen(
query_lens, sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
):
sparse_call.update(
query_lens=query_lens.clone(),
seq_lens=sparse_seq_lens.clone(),
query_start_loc=sparse_query_start_loc.clone(),
num_tokens=num_tokens,
)
return torch.ones(num_tokens, dtype=torch.int32)
monkeypatch.setattr(
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
)
fake_aiter = _FakeAiter("aiter")
fake_aiter.get_mla_metadata_v1 = Mock()
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: None),
)
result = builder.build(0, common_metadata)
assert result.prefill is metadata.prefill
assert result.num_prefills == 1
assert result.req_id_per_token.tolist() == [0]
assert result.qo_indptr.numel() == 2
assert result.paged_kv_indptr.numel() == 2
assert sparse_call["num_tokens"] == 1
assert sparse_call["query_lens"].tolist() == [1]
assert sparse_call["seq_lens"].tolist() == [16]
assert sparse_call["query_start_loc"].tolist() == [0, 1]
def test_long_prefill_shares_full_sparse_metadata_with_decode(monkeypatch):
"""A prefill longer than top-k must remain in shared sparse metadata.
Dense MHA is only selected when the prefill sequence fits within the
sparse top-k window. Once it exceeds that window, both decode and prefill
are handled by sparse MQA, so every query token must be represented by the
same sparse paging metadata.
"""
builder = _make_builder()
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
)
# One decode token followed by a five-token prefill. topk_tokens is four,
# so the prefill must not be split onto the dense-MHA path.
query_start_loc = torch.tensor([0, 1, 6], dtype=torch.int32)
seq_lens = torch.tensor([16, 5], dtype=torch.int32)
common_metadata = SimpleNamespace(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
max_query_len=5,
max_seq_len=16,
)
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
num_reqs=2,
max_query_len=5,
max_seq_len=16,
num_actual_tokens=6,
query_start_loc=query_start_loc,
slot_mapping=torch.arange(6),
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
req_id_per_token=torch.tensor([0, 1, 1, 1, 1, 1], dtype=torch.int32),
topk_tokens=4,
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
seq_lens=seq_lens,
prefill_max_seq_len=5,
prefill=SimpleNamespace(use_dense_mha=False),
)
monkeypatch.setattr(
sparse_mod.SparseMLACommonMetadataBuilder,
"build",
lambda *args, **kwargs: metadata,
)
sparse_call: dict[str, torch.Tensor] = {}
def fake_generate_sparse_seqlen(
query_lens, sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
):
sparse_call.update(
query_lens=query_lens.clone(),
seq_lens=sparse_seq_lens.clone(),
query_start_loc=sparse_query_start_loc.clone(),
num_tokens=num_tokens,
)
return torch.ones(num_tokens, dtype=torch.int32)
monkeypatch.setattr(
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
)
fake_aiter = _FakeAiter("aiter")
get_persistent_metadata = Mock()
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: None),
)
result = builder.build(0, common_metadata)
assert result.num_decodes == 1
assert result.num_prefills == 1
assert result.req_id_per_token.tolist() == [0, 1, 1, 1, 1, 1]
assert result.qo_indptr.numel() == 7
assert result.paged_kv_indptr.numel() == 7
assert result.paged_kv_indices.numel() == 6 * result.topk_tokens
assert sparse_call["num_tokens"] == 6
assert sparse_call["query_lens"].tolist() == [1, 5]
assert sparse_call["seq_lens"].tolist() == [16, 5]
assert sparse_call["query_start_loc"].tolist() == [0, 1, 6]
get_persistent_metadata.assert_called_once()
assert result.work_meta_data is builder._mla_work_meta_data
assert result.reduce_indptr is builder._mla_reduce_indptr
def test_forward_mqa_slices_full_batch_metadata_to_query_tokens(monkeypatch):
"""The sparse kernel inputs must have the same token extent as its query.
In a mixed dense-prefill batch, ``forward_mqa`` receives only decode query
tokens while some metadata buffers may represent the full batch. This
guards the slicing that prevents the index conversion from writing beyond
the decode-sized output.
"""
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
impl.kv_cache_dtype = "auto"
impl.num_heads = 4
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
forward_mla = Mock(return_value=(expected_output, None))
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
metadata = SimpleNamespace(
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
block_size=1,
topk_tokens=4,
)
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
captured = {}
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
captured["req_ids"] = req_ids.clone()
captured["token_indices"] = token_indices.clone()
monkeypatch.setattr(
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
)
monkeypatch.setattr(
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
)
output, lse = impl.forward_mqa(
query,
torch.zeros(1, 1, 576, dtype=torch.bfloat16),
metadata,
SimpleNamespace(),
)
assert output is expected_output
assert lse is None
assert captured["req_ids"].tolist() == [0, 1]
assert captured["token_indices"].shape == (2, 4)
forward_mla.assert_called_once()
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_forward_mqa_slices_decode_subset_for_bf16_and_fp8(monkeypatch, kv_cache_dtype):
"""Both cache dtypes must slice mixed-batch metadata to decode queries."""
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
impl.kv_cache_dtype = kv_cache_dtype
impl.num_heads = 4
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
forward_mla = Mock(return_value=(expected_output, None))
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
metadata = SimpleNamespace(
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
block_size=1,
topk_tokens=4,
)
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
kv_cache = torch.zeros(1, 1, 576, dtype=torch.uint8)
captured = {}
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
captured["req_ids"] = req_ids.clone()
captured["token_indices"] = token_indices.clone()
monkeypatch.setattr(
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
)
monkeypatch.setattr(
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
)
if kv_cache_dtype == "fp8":
fp8_dtype = current_platform.fp8_dtype()
monkeypatch.setattr(
sparse_mod.ops,
"scaled_fp8_quant",
lambda q, scale: (q.to(fp8_dtype), None),
)
output, lse = impl.forward_mqa(
query,
kv_cache,
metadata,
SimpleNamespace(_q_scale=torch.tensor(1.0)),
)
assert output is expected_output
assert lse is None
assert captured["req_ids"].tolist() == [0, 1]
assert captured["token_indices"].shape == (2, 4)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("length_relation", ["short", "long"])
@pytest.mark.parametrize("workload", ["prefill", "mixed", "decode"])
def test_sparse_metadata_workload_length_dtype_matrix(
monkeypatch, workload, length_relation, kv_cache_dtype
):
"""Cover routing and persistent metadata across the ROCm support matrix.
The matrix crosses pure prefill, mixed prefill/decode, and decode-only
workloads with sequence lengths below/above top-k and BF16/FP8 KV cache.
"""
builder = _make_builder()
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
)
if kv_cache_dtype == "fp8":
builder._mla_q_dtype = current_platform.fp8_dtype()
builder._mla_kv_dtype = current_platform.fp8_dtype()
is_long = length_relation == "long"
prefill_len = 5 if is_long else 3
decode_seq_len = 8 if is_long else 3
if workload == "prefill":
query_lens = [prefill_len]
seq_lens_list = [prefill_len]
num_decodes, num_prefills = 0, 1
elif workload == "mixed":
query_lens = [1, prefill_len]
seq_lens_list = [decode_seq_len, prefill_len]
num_decodes, num_prefills = 1, 1
else:
query_lens = [1, 1]
seq_lens_list = [decode_seq_len, decode_seq_len + 1]
num_decodes, num_prefills = 2, 0
query_start_loc = torch.tensor(
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
)
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32)
num_tokens = sum(query_lens)
common_metadata = SimpleNamespace(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
max_query_len=max(query_lens),
max_seq_len=max(seq_lens_list),
)
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
num_reqs=len(query_lens),
max_query_len=max(query_lens),
max_seq_len=max(seq_lens_list),
num_actual_tokens=num_tokens,
query_start_loc=query_start_loc,
slot_mapping=torch.arange(num_tokens),
block_table=torch.arange(len(query_lens) * 8, dtype=torch.int32).view(
len(query_lens), 8
),
req_id_per_token=torch.repeat_interleave(
torch.arange(len(query_lens), dtype=torch.int32),
torch.tensor(query_lens),
),
topk_tokens=4,
num_decodes=num_decodes,
num_prefills=num_prefills,
num_decode_tokens=num_decodes,
seq_lens=seq_lens,
prefill_max_seq_len=prefill_len if num_prefills else 0,
prefill=SimpleNamespace(use_dense_mha=not is_long) if num_prefills else None,
)
monkeypatch.setattr(
sparse_mod.SparseMLACommonMetadataBuilder,
"build",
lambda *args, **kwargs: metadata,
)
monkeypatch.setattr(
sparse_mod,
"generate_sparse_seqlen_triton",
lambda *args, **kwargs: torch.ones(args[4], dtype=torch.int32),
)
get_persistent_metadata = Mock()
fake_aiter = _FakeAiter("aiter")
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: None),
)
result = builder.build(0, common_metadata)
uses_dense_prefill = workload in ("prefill", "mixed") and not is_long
expected_sparse_tokens = num_decodes if uses_dense_prefill else num_tokens
assert result.req_id_per_token.numel() == expected_sparse_tokens
assert result.qo_indptr.numel() == expected_sparse_tokens + 1
assert result.paged_kv_indptr.numel() == expected_sparse_tokens + 1
uses_fallback = workload == "prefill" and is_long and kv_cache_dtype == "auto"
if expected_sparse_tokens == 0 or uses_fallback:
get_persistent_metadata.assert_not_called()
else:
get_persistent_metadata.assert_called_once()
if uses_fallback or expected_sparse_tokens == 0:
assert result.work_meta_data is None
assert result.reduce_indptr is None
else:
assert result.work_meta_data is builder._mla_work_meta_data
assert result.reduce_indptr is builder._mla_reduce_indptr
@@ -155,8 +155,86 @@ def test_rocm_sparse_triton_route(
)
@pytest.mark.parametrize(
("gfx942", "fp8", "num_prefills", "num_decodes", "prefill_max_seq_len", "route"),
[
pytest.param(True, False, 1, 0, 129, "triton", id="above-topk"),
pytest.param(False, False, 1, 0, 129, "aiter", id="gfx950"),
pytest.param(True, True, 1, 0, 129, "aiter", id="fp8"),
pytest.param(True, False, 0, 2, 129, "aiter", id="decode-only"),
pytest.param(True, False, 1, 1, 129, "aiter", id="mixed"),
pytest.param(True, False, 1, 0, 128, "aiter", id="equal-topk"),
pytest.param(True, False, 1, 0, 127, "aiter", id="below-topk"),
pytest.param(True, False, 0, 0, 129, "aiter", id="empty"),
pytest.param(
True, False, 1, 0, None, "missing", id="required-prefill-metadata"
),
],
)
def test_rocm_long_bf16_prefill_dispatch(
monkeypatch, gfx942, fp8, num_prefills, num_decodes, prefill_max_seq_len, route
):
"""Check the new dispatch gate using CPU tensors and intercepted kernels."""
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942)
class KernelSelected(Exception):
pass
def triton_kernel(**kwargs):
raise KernelSelected("triton")
def aiter_kernel(*args, **kwargs):
raise KernelSelected("aiter")
monkeypatch.setattr(sparse_mod, "rocm_sparse_attn_prefill", triton_kernel)
monkeypatch.setattr(sparse_mod.rocm_aiter_ops, "mla_decode_fwd", aiter_kernel)
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
impl.num_heads = 64
impl.kv_lora_rank = 512
impl.kv_cache_dtype = "fp8" if fp8 else "auto"
impl.scale = 576**-0.5
impl.sinks = None
dtype = torch.float8_e4m3fn if fp8 else torch.bfloat16
q = torch.empty(2, 64, 576, dtype=dtype)
kv = torch.empty(4, 1, 576, dtype=dtype)
metadata = SimpleNamespace(
attn_out_dtype=torch.bfloat16,
num_prefills=num_prefills,
num_decodes=num_decodes,
num_decode_tokens=num_decodes,
max_query_len=2,
prefill_max_seq_len=prefill_max_seq_len,
topk_tokens=128,
work_meta_data=None,
qo_indptr=torch.arange(3, dtype=torch.int32),
paged_kv_indices=torch.empty(0, dtype=torch.int32),
paged_kv_indptr=torch.zeros(3, dtype=torch.int32),
paged_kv_last_page_len=torch.ones(2, dtype=torch.int32),
)
# Irrelevant routes must not inspect prefill-only metadata.
if (
not gfx942
or fp8
or num_decodes
or not num_prefills
or prefill_max_seq_len is None
):
del metadata.prefill_max_seq_len
del metadata.topk_tokens
layer = SimpleNamespace(_q_scale=1.0, _k_scale=1.0)
error = AttributeError if route == "missing" else KernelSelected
match = "prefill_max_seq_len" if route == "missing" else f"^{route}$"
with pytest.raises(error, match=match):
impl._forward_mla(layer, q, kv, metadata)
@pytest.mark.parametrize("gfx942", [False, True])
@pytest.mark.parametrize("num_heads", [8, 12])
def test_rocm_sparse_triton_route_preserves_padded_sinks(monkeypatch, num_heads):
def test_rocm_sparse_triton_route_preserves_padded_sinks(
monkeypatch, num_heads, gfx942
):
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942)
captured = {}
def fake_rocm_sparse_attn_prefill(**kwargs):
+246 -29
View File
@@ -47,13 +47,12 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.platforms import current_platform
# TODO: Integrate ROCMAiterMLASparseBackend for ROCm.
# The ROCm sparse MLA backend (rocm_aiter_mla_sparse.py) has a compatible
# forward_mqa interface but needs validation on ROCm hardware.
if not current_platform.is_cuda():
# This module's end-to-end sparse-backend correctness test supports CUDA and
# ROCm. The remaining tests exercise CUDA-specific FlashMLA/FlashInfer APIs
# and are skipped by the autouse fixture below when collecting on ROCm.
if not (current_platform.is_cuda() or current_platform.is_rocm()):
pytest.skip(
"Sparse MLA backend tests currently only support CUDA. "
"ROCm support requires integrating ROCMAiterMLASparseBackend.",
"Sparse MLA backend tests require CUDA or ROCm.",
allow_module_level=True,
)
@@ -65,14 +64,16 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
)
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backends.mla import (
flashattn_mla_sparse as flashattn_sparse_module,
)
from vllm.v1.attention.backends.mla import index_group as index_group_module
from vllm.v1.attention.backends.mla.flashattn_mla_sparse import (
FlashAttnMLASparseImpl,
FlashAttnMLASparseMetadataBuilder,
)
if current_platform.is_cuda():
from vllm.v1.attention.backends.mla import (
flashattn_mla_sparse as flashattn_sparse_module,
)
from vllm.v1.attention.backends.mla.flashattn_mla_sparse import (
FlashAttnMLASparseImpl,
FlashAttnMLASparseMetadataBuilder,
)
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseImpl,
FlashInferMLASparseTRTLLMBackend,
@@ -111,9 +112,43 @@ from vllm.v1.hisparse.runtime import (
)
from vllm.v1.hisparse.types import SparseKVRowMirror
if current_platform.is_rocm():
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
SPARSE_BACKENDS = [ROCMAiterMLASparseBackend]
SPARSE_BACKEND_IDS = ["ROCMAiterMLA"]
SPARSE_KV_CACHE_DTYPES = ["auto", "fp8"]
SPARSE_BLOCK_SIZES = [64]
else:
SPARSE_BACKENDS = [FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend]
SPARSE_BACKEND_IDS = ["FlashMLA", "FlashInferTRTLLM"]
SPARSE_KV_CACHE_DTYPES = ["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"]
SPARSE_BLOCK_SIZES = [32, 64]
@pytest.fixture(autouse=True)
def _skip_cuda_specific_sparse_mla_tests_on_rocm(request):
"""Skip tests whose helpers exercise CUDA-only sparse MLA APIs."""
rocm_portable_tests = {
"test_sparse_backend_decode_correctness",
"test_sparse_backend_prefill_correctness",
"test_rocm_fused_v32_dense_prefill_matches_mqa",
"test_rocm_long_bf16_prefill_avoids_nonpersistent_aiter",
}
if (
current_platform.is_rocm()
and getattr(request.node, "originalname", request.node.name)
not in rocm_portable_tests
):
pytest.skip("This sparse MLA test exercises a CUDA-specific API")
SPARSE_BACKEND_BATCH_SPECS = {
name: BATCH_SPECS[name]
for name in [
"small_decode",
"mixed_small",
"mixed_medium",
"small_prefill",
@@ -128,10 +163,61 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_prefill"] = BatchSpec(
SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
seq_lens=[256] * 2, query_lens=[256] * 2
)
SPARSE_BACKEND_BATCH_SPECS["large_decode_over_topk"] = BatchSpec(
seq_lens=[256, 384], query_lens=[1, 1]
)
DEVICE_TYPE = current_platform.device_type
@pytest.mark.parametrize(
("batch_name", "tensor_parallel_size"),
[
("large_q_prefill", 2),
("large_q_pure_prefill", 4),
("large_q_pure_prefill", 1),
],
)
def test_rocm_long_bf16_prefill_avoids_nonpersistent_aiter(
default_vllm_config,
dist_init,
workspace_init,
monkeypatch,
batch_name,
tensor_parallel_size,
):
"""MI300 long BF16 prefills must remain correct without AITER ASM dispatch."""
if not current_platform.is_rocm():
pytest.skip("ROCm sparse MLA regression")
from vllm._aiter_ops import rocm_aiter_ops
# Exercise the gfx942 routing decision on other ROCm devices too;
# the numerical kernel still compiles for the actual device.
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: True)
def unsupported_aiter(*args, **kwargs):
pytest.fail("Long gfx942 BF16 prefill reached nonpersistent AITER")
monkeypatch.setattr(rocm_aiter_ops, "mla_decode_fwd", unsupported_aiter)
test_sparse_backend_decode_correctness(
default_vllm_config=default_vllm_config,
dist_init=dist_init,
backend_cls=ROCMAiterMLASparseBackend,
batch_name=batch_name,
kv_cache_dtype="auto",
tensor_parallel_size=tensor_parallel_size,
block_size=64,
workspace_init=workspace_init,
q_scale=1.0,
k_scale=1.0,
total_num_heads=128,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
monkeypatch=monkeypatch,
)
def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
"""Weight absorption must not change the model's attention temperature."""
model_scale = 256**-0.5
@@ -403,16 +489,13 @@ def _quantize_dequantize_nvfp4_ds_mla(
@pytest.mark.parametrize(
"backend_cls",
[FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend],
ids=["FlashMLA", "FlashInferTRTLLM"],
SPARSE_BACKENDS,
ids=SPARSE_BACKEND_IDS,
)
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
@pytest.mark.parametrize(
"kv_cache_dtype",
["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"],
)
@pytest.mark.parametrize("kv_cache_dtype", SPARSE_KV_CACHE_DTYPES)
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
@pytest.mark.parametrize("block_size", [32, 64])
@pytest.mark.parametrize("block_size", SPARSE_BLOCK_SIZES)
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
@pytest.mark.parametrize(
("total_num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
@@ -525,6 +608,12 @@ def test_sparse_backend_decode_correctness(
"attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
},
)
# This shared correctness test constructs a sparse SDPA reference for every
# query token. ROCm normally routes short prefills to dense MHA, whose
# output intentionally differs from that sparse reference. Keep this test
# on the sparse-MQA path; dense-prefill routing has dedicated ROCm tests.
if current_platform.is_rocm():
vllm_config.attention_config.sparse_mla_force_mqa = True
model_config = vllm_config.model_config
model_config.hf_text_config = SimpleNamespace(
index_topk=topk_tokens,
@@ -737,7 +826,7 @@ def test_sparse_backend_decode_correctness(
vllm_config=vllm_config,
)
vllm_config.compilation_config.static_forward_context["placeholder"] = (
SimpleNamespace(prefill_backend=prefill_backend)
SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace())
)
builder_cls = backend_cls.get_builder_cls()
@@ -1384,11 +1473,13 @@ PREFILL_BATCH_SPECS = {
@pytest.mark.skipif(
torch.cuda.get_device_capability()[0] < 10,
current_platform.is_cuda() and torch.cuda.get_device_capability()[0] < 10,
reason="Sparse MLA forward_mha requires FA4 (SM100+)",
)
@pytest.mark.parametrize("batch_name", list(PREFILL_BATCH_SPECS.keys()))
@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
@pytest.mark.parametrize(
"kv_cache_dtype", (["auto", "fp8"] if current_platform.is_rocm() else ["auto"])
)
@pytest.mark.parametrize(
("num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
[
@@ -1408,11 +1499,19 @@ def test_sparse_backend_prefill_correctness(
v_head_dim,
workspace_init,
):
"""Test dense and masked MHA across supported sparse MLA dimensions."""
backend_cls = FlashMLASparseBackend
"""Test the normal dense forward_mha path for sparse MLA short prefill."""
backend_cls = (
ROCMAiterMLASparseBackend
if current_platform.is_rocm()
else FlashMLASparseBackend
)
if current_platform.is_rocm() and (
batch_name.startswith("masked_mha") or qk_nope_head_dim != 128
):
pytest.skip("ROCm dense prefill requires the DeepSeek MLA layout")
batch_spec = PREFILL_BATCH_SPECS[batch_name]
device = torch.device("cuda")
device = torch.device(DEVICE_TYPE)
dtype = torch.bfloat16
block_size = 64
@@ -1579,7 +1678,7 @@ def test_sparse_backend_prefill_correctness(
vllm_config=vllm_config,
)
vllm_config.compilation_config.static_forward_context["placeholder"] = (
SimpleNamespace(prefill_backend=prefill_backend)
SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace())
)
builder_cls = backend_cls.get_builder_cls()
@@ -1653,7 +1752,10 @@ def test_sparse_backend_prefill_correctness(
assert out_buffer.shape == ref_output.shape
assert torch.isfinite(out_buffer).all(), "Non-finite values in output"
torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01)
if kv_cache_dtype.startswith("fp8"):
torch.testing.assert_close(out_buffer, ref_output, rtol=0.065, atol=0.05)
else:
torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01)
@pytest.mark.parametrize(
@@ -3057,7 +3159,7 @@ def test_hisparse_mixed_batch_bf16_row_split(
vllm_config=vllm_config,
)
vllm_config.compilation_config.static_forward_context["placeholder"] = (
SimpleNamespace(prefill_backend=prefill_backend)
SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace())
)
builder_cls = FlashMLASparseBackend.get_builder_cls()
@@ -4264,3 +4366,118 @@ def test_sparse_mla_common_impl_resolves_buffer_lazily():
assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
"topk_indices_buffer must stay a lazily-resolved property"
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm model implementation")
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_rocm_fused_v32_dense_prefill_matches_mqa(
tmp_path, vllm_runner, kv_cache_dtype
):
"""Fused normalization, RoPE and mixed dispatch preserve generation/logprobs.
Chunking unequal prompts exercises prefill alongside decode. The two long
prompts straddle the dense-prefill threshold. No pretrained weights or
tokenizer download is needed.
"""
import json
from vllm import SamplingParams
config = {
"architectures": ["DeepseekV32ForCausalLM"],
"attention_bias": False,
"attention_dropout": 0.0,
"bos_token_id": 0,
"eos_token_id": 1,
"ep_size": 1,
"first_k_dense_replace": 1,
"hidden_act": "silu",
"hidden_size": 2048,
"index_head_dim": 128,
"index_n_heads": 64,
"index_topk": 2048,
"initializer_range": 0.02,
"intermediate_size": 4096,
"kv_lora_rank": 512,
"max_position_embeddings": 8192,
"model_type": "deepseek_v32",
"moe_intermediate_size": 512,
"moe_layer_freq": 1,
"n_group": 1,
"n_routed_experts": 8,
"n_shared_experts": 1,
"norm_topk_prob": True,
"num_attention_heads": 32,
"num_experts_per_tok": 2,
"num_hidden_layers": 2,
"num_key_value_heads": 32,
"num_nextn_predict_layers": 0,
"q_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"rms_norm_eps": 1e-06,
"rope_scaling": {
"beta_fast": 32,
"beta_slow": 1,
"factor": 40,
"mscale": 1.0,
"mscale_all_dim": 1.0,
"original_max_position_embeddings": 4096,
"type": "yarn",
},
"rope_theta": 10000,
"routed_scaling_factor": 2.5,
"scoring_func": "sigmoid",
"tie_word_embeddings": False,
"topk_group": 1,
"topk_method": "noaux_tc",
"torch_dtype": "bfloat16",
"transformers_version": "4.44.2",
"use_cache": True,
"v_head_dim": 128,
"vocab_size": 4096,
}
(tmp_path / "config.json").write_text(json.dumps(config))
prompts = [
{"prompt_token_ids": [2 + (i * 17 + j * 13) % 4000 for i in range(n)]}
for j, n in enumerate([64, 128, 2048, 2049])
]
sampling = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True, logprobs=5)
results = []
for force_mqa in [True, False]:
with vllm_runner(
str(tmp_path),
load_format="dummy",
skip_tokenizer_init=True,
dtype="bfloat16",
kv_cache_dtype=kv_cache_dtype,
seed=42,
max_model_len=4096,
max_num_seqs=4,
max_num_batched_tokens=256,
block_size=64,
enable_chunked_prefill=True,
enable_prefix_caching=False,
enforce_eager=True,
gpu_memory_utilization=0.25,
attention_config={
"backend": "ROCM_AITER_MLA_SPARSE",
"sparse_mla_force_mqa": force_mqa,
},
) as runner:
outputs = runner.llm.generate(prompts, sampling, use_tqdm=False)
results.append([request.outputs[0] for request in outputs])
for reference, actual in zip(*results):
assert len(actual.token_ids) == 16
assert actual.token_ids == reference.token_ids
assert actual.logprobs is not None and reference.logprobs is not None
for token, expected, observed in zip(
actual.token_ids, reference.logprobs, actual.logprobs
):
torch.testing.assert_close(
torch.tensor(observed[token].logprob),
torch.tensor(expected[token].logprob),
rtol=0.01,
atol=0.01,
)
+33 -2
View File
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer
@@ -179,6 +182,8 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
else:
attn_metadata = attn_metadata_raw
attn_metadata = cast(MLACommonMetadata | None, attn_metadata)
slot_mapping = forward_context.slot_mapping
assert isinstance(slot_mapping, dict)
mla_slot = slot_mapping.get(self.layer_name)
@@ -213,6 +218,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
mla_kv_cache = self.kv_cache
mla_k_scale = self._k_scale
use_mha = attn_metadata is not None and self._use_sparse_mha(attn_metadata)
kv_c_out = torch.empty_like(kv_c) if use_mha else None
k_pe_out = torch.empty_like(k_pe) if use_mha else None
q_c = fused_norm_rope(
positions,
q_c,
@@ -237,9 +246,18 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
mla_k_scale=mla_k_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
kv_c_out=kv_c_out,
k_pe_out=k_pe_out,
)
ql_nope, q_pe = self._compute_ql_nope(q_c)
if use_mha:
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split(
[self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1
)
ql_nope = q_nope
else:
ql_nope, q_pe = self._compute_ql_nope(q_c)
if self.indexer is not None and not self.skip_topk:
index_q = self.indexer.wq_b(q_c)[0]
@@ -260,7 +278,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
indexer_n_head_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
quantize_mqa=self._fp8_kv,
quantize_mqa=self._fp8_kv and not use_mha,
)
if self.indexer is not None and not self.skip_topk:
@@ -270,6 +288,19 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
output.zero_()
return
if use_mha:
assert kv_c_out is not None and k_pe_out is not None
mha_q = torch.cat((q_nope, mqa_q), dim=-1)
self.forward_impl(
mha_q,
kv_c_out,
k_pe_out.unsqueeze(1),
self.kv_cache,
attn_metadata,
output,
)
return
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
kv_cache = self.kv_cache
if self._fp8_kv_needs_view:
@@ -12,11 +12,10 @@ from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
get_mla_dims,
)
from vllm.model_executor.layers.attention.sparse_mla_attention import (
SharedTopkIndicesBuffer,
SparseMLACommonImpl,
SparseMLACommonMetadata,
SparseMLACommonMetadataBuilder,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -26,10 +25,7 @@ from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionLayer,
AttentionMetadata,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MLAAttentionImpl,
MultipleOf,
)
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
@@ -39,7 +35,6 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
align_blocks_to_rows,
flat_kv_row_view,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_sparse_attn_decode_bf16,
rocm_sparse_attn_prefill,
@@ -76,6 +71,18 @@ def _use_rocm_sparse_triton(
)
def _is_long_bf16_prefill(
kv_dtype: torch.dtype,
metadata: "ROCMAiterMLASparseMetadata",
) -> bool:
return (
kv_dtype == torch.bfloat16
and metadata.num_decodes == 0
and metadata.num_prefills > 0
and metadata.prefill_max_seq_len > metadata.topk_tokens
)
@triton.jit
def _fit_kpool_indices_kernel(
token_indices_ptr, # int32 [num_tokens, NUM_TOPK_TOKENS + TAIL_WIDTH]
@@ -239,6 +246,10 @@ def triton_convert_req_index_to_global_index(
assert req_id.dtype == torch.int32
assert block_table.dtype == torch.int32
assert token_indices.dtype == torch.int32
assert req_id.shape[0] == token_indices.shape[0], (
f"req_id ({req_id.shape[0]}) and token_indices ({token_indices.shape[0]}) "
"must cover the same tokens"
)
assert token_indices.shape[1] == NUM_TOPK_TOKENS
assert NUM_TOPK_TOKENS % BLOCK_N == 0, (
f"NUM_TOPK_TOKENS ({NUM_TOPK_TOKENS}) must be divisible byBLOCK_N ({BLOCK_N})"
@@ -390,35 +401,13 @@ class ROCMAiterMLASparseBackend(AttentionBackend):
return on_mi3xx()
@dataclass
class ROCMAiterMLASparseMetadata(AttentionMetadata):
num_reqs: int
max_query_len: int
max_seq_len: int
num_actual_tokens: int # Number of tokens excluding padding.
query_start_loc: torch.Tensor
slot_mapping: torch.Tensor
block_table: torch.Tensor
req_id_per_token: torch.Tensor
qo_indptr: torch.Tensor
paged_kv_last_page_len: torch.Tensor
paged_kv_indices: torch.Tensor
paged_kv_indptr: torch.Tensor
attn_out_dtype: torch.dtype
block_size: int = 1
topk_tokens: int = 2048
# Fields read by the shared MLA forward. This impl has no dense-MHA prefill
# path (supports_dense_mha_prefill=False), so it always runs the MQA path;
num_decodes: int = 0
num_prefills: int = 0
num_decode_tokens: int = 0
prefill_max_seq_len: int = 0
prefill: object = None
@dataclass(kw_only=True)
class ROCMAiterMLASparseMetadata(SparseMLACommonMetadata):
qo_indptr: torch.Tensor | None = None
paged_kv_last_page_len: torch.Tensor | None = None
paged_kv_indices: torch.Tensor | None = None
paged_kv_indptr: torch.Tensor | None = None
attn_out_dtype: torch.dtype | None = None
# Persistent MLA metadata (only populated when persistent mode is enabled,
# i.e. when the aiter sparse decode kernel supports work-stealing splits).
@@ -432,9 +421,10 @@ class ROCMAiterMLASparseMetadata(AttentionMetadata):
@dataclass
class ROCMAiterMLASparseMetadataBuilder(
AttentionMetadataBuilder[ROCMAiterMLASparseMetadata]
SparseMLACommonMetadataBuilder[ROCMAiterMLASparseMetadata]
):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
metadata_cls = ROCMAiterMLASparseMetadata
def __init__(
self,
@@ -443,20 +433,15 @@ class ROCMAiterMLASparseMetadataBuilder(
vllm_config: VllmConfig,
device: torch.device,
):
self.kv_cache_spec = kv_cache_spec
self.model_config = vllm_config.model_config
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
self.model_dtype = vllm_config.model_config.dtype
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
parallel_config = vllm_config.parallel_config
self.device = device
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.vllm_config = vllm_config
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
self.mla_dims = get_mla_dims(self.model_config)
self.topk_tokens = vllm_config.model_config.hf_text_config.index_topk
attention_context = vllm_config.compilation_config.static_forward_context
# Sink decode must use AITER's nonpersistent path. In particular,
# gfx942 has no persistent+LSE kernel, and its metadata heuristic
@@ -521,9 +506,12 @@ class ROCMAiterMLASparseMetadataBuilder(
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
kv_cache_dtype_str = "fp8"
q_dtype = dtypes.fp8
else:
kv_cache_dtype_str = "bf16"
kv_dtype = dtypes.d_dtypes.get(kv_cache_dtype_str, dtypes.bf16)
self._mla_q_dtype = q_dtype
self._mla_kv_dtype = kv_dtype
(
(work_meta_data_size, work_meta_data_type),
@@ -587,13 +575,19 @@ class ROCMAiterMLASparseMetadataBuilder(
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> ROCMAiterMLASparseMetadata:
num_tokens = common_attn_metadata.num_actual_tokens
(num_decodes, num_prefills, num_decode_tokens, _) = split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold or 1,
metadata = super().build(
common_prefix_len, common_attn_metadata, fast_build=fast_build
)
use_dense_prefill = (
metadata.prefill is not None and metadata.prefill.use_dense_mha
)
num_reqs = metadata.num_decodes if use_dense_prefill else metadata.num_reqs
num_tokens = int(common_attn_metadata.query_start_loc_cpu[num_reqs].item())
num_decodes = metadata.num_decodes
num_prefills = metadata.num_prefills
num_decode_tokens = metadata.num_decode_tokens
starts = np.asarray(common_attn_metadata.query_start_loc_cpu, dtype=np.int32)
seg_lengths = np.diff(starts)
seg_lengths = np.diff(starts[: num_reqs + 1])
req_id_per_token = np.repeat(
np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths
)
@@ -616,15 +610,13 @@ class ROCMAiterMLASparseMetadataBuilder(
self.req_id_per_token_buffer[:new_req_extent].copy_(
np_to_pinned_tensor(req_id_per_token), non_blocking=True
)
query_lens = (
common_attn_metadata.query_start_loc[1:]
- common_attn_metadata.query_start_loc[:-1]
)
seq_lens = common_attn_metadata.seq_lens
query_start_loc = common_attn_metadata.query_start_loc[: num_reqs + 1]
query_lens = query_start_loc[1:] - query_start_loc[:-1]
seq_lens = common_attn_metadata.seq_lens[:num_reqs]
sparse_seqlen = generate_sparse_seqlen_triton(
query_lens,
seq_lens,
common_attn_metadata.query_start_loc,
query_start_loc,
self.topk_tokens,
num_tokens,
common_attn_metadata.max_query_len,
@@ -663,8 +655,13 @@ class ROCMAiterMLASparseMetadataBuilder(
reduce_indptr = None
reduce_final_map = None
reduce_partial_map = None
if self._use_persistent_metadata and not use_triton_sparse:
num_reqs = common_attn_metadata.num_reqs
is_long_pure_bf16_prefill = _is_long_bf16_prefill(self._mla_kv_dtype, metadata)
if (
self._use_persistent_metadata
and not use_triton_sparse
and not is_long_pure_bf16_prefill
and num_tokens > 0
):
with gpu_sync_allowed():
seq_lens_cpu = common_attn_metadata.seq_lens[:num_reqs].cpu().numpy()
clamped_seq_lens = np.minimum(
@@ -708,6 +705,8 @@ class ROCMAiterMLASparseMetadataBuilder(
uni_seqlen_qo=1,
fast_mode=True,
max_split_per_batch=max_split_per_batch,
dtype_q=self._mla_q_dtype,
dtype_kv=self._mla_kv_dtype,
)
torch.cuda.current_stream(self.device).synchronize()
self._prev_metadata_key = metadata_key
@@ -718,32 +717,18 @@ class ROCMAiterMLASparseMetadataBuilder(
reduce_final_map = self._mla_reduce_final_map
reduce_partial_map = self._mla_reduce_partial_map
metadata = ROCMAiterMLASparseMetadata(
num_reqs=common_attn_metadata.num_reqs,
max_query_len=common_attn_metadata.max_query_len,
max_seq_len=common_attn_metadata.max_seq_len,
num_actual_tokens=common_attn_metadata.num_actual_tokens,
query_start_loc=common_attn_metadata.query_start_loc,
slot_mapping=common_attn_metadata.slot_mapping,
block_table=common_attn_metadata.block_table_tensor,
req_id_per_token=req_id_per_token,
block_size=self.kv_cache_spec.block_size,
attn_out_dtype=self.model_dtype,
topk_tokens=self.topk_tokens,
num_decodes=num_decodes,
num_prefills=num_prefills,
num_decode_tokens=num_decode_tokens,
qo_indptr=qo_indptr,
paged_kv_last_page_len=paged_kv_last_page_len,
paged_kv_indices=paged_kv_indices,
paged_kv_indptr=paged_kv_indptr,
work_meta_data=work_meta_data,
work_indptr=work_indptr,
work_info_set=work_info_set,
reduce_indptr=reduce_indptr,
reduce_final_map=reduce_final_map,
reduce_partial_map=reduce_partial_map,
)
metadata.attn_out_dtype = self.model_dtype
metadata.req_id_per_token = req_id_per_token
metadata.qo_indptr = qo_indptr
metadata.paged_kv_last_page_len = paged_kv_last_page_len
metadata.paged_kv_indices = paged_kv_indices
metadata.paged_kv_indptr = paged_kv_indptr
metadata.work_meta_data = work_meta_data
metadata.work_indptr = work_indptr
metadata.work_info_set = work_info_set
metadata.reduce_indptr = reduce_indptr
metadata.reduce_final_map = reduce_final_map
metadata.reduce_partial_map = reduce_partial_map
return metadata
@@ -776,11 +761,7 @@ def reference_mla_sparse_prefill(
return (result, lse)
class ROCMAiterMLASparseImpl(
MLAAttentionImpl[ROCMAiterMLASparseMetadata], SharedTopkIndicesBuffer
):
is_sparse = True
supports_dense_mha_prefill = False
class ROCMAiterMLASparseImpl(SparseMLACommonImpl[ROCMAiterMLASparseMetadata]):
supports_dcp = False
use_aiter_sparse_mla = False
@@ -803,10 +784,6 @@ class ROCMAiterMLASparseImpl(
) -> None:
AiterMLAHelper.check_num_heads_validity(num_heads)
self.num_heads = num_heads
self.head_size = head_size
self.scale = float(scale)
self.num_kv_heads = num_kv_heads
sinks = mla_args.pop("sinks", None)
if sinks is not None:
if sinks.dtype != torch.float32:
@@ -821,10 +798,24 @@ class ROCMAiterMLASparseImpl(
if not sinks.is_contiguous():
raise ValueError("ROCm AITER MLA sinks must be contiguous")
self.sinks: torch.Tensor | None = sinks
self.kv_cache_dtype = kv_cache_dtype
self.kv_lora_rank: int = mla_args["kv_lora_rank"]
super().__init__(
num_heads,
head_size,
scale,
num_kv_heads,
alibi_slopes,
sliding_window,
kv_cache_dtype,
logits_soft_cap,
attn_type,
kv_sharing_target_layer_name,
indexer=indexer,
topk_indices_buffer=topk_indices_buffer,
**mla_args,
)
self.softmax_scale = scale
self.init_topk_indices_buffer(indexer, topk_indices_buffer)
self.supports_dense_mha_prefill = sinks is None # type: ignore[misc]
self.masked_mha_available = False
vllm_config = get_current_vllm_config()
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
@@ -894,14 +885,26 @@ class ROCMAiterMLASparseImpl(
)
)
if triton_sink_fallback or _use_rocm_sparse_triton(
kv_cache_dtype=self.kv_cache_dtype,
head_size=q.shape[-1],
kv_lora_rank=self.kv_lora_rank,
num_prefills=attn_metadata.num_prefills,
num_decodes=attn_metadata.num_decodes,
num_decode_tokens=attn_metadata.num_decode_tokens,
max_query_len=attn_metadata.max_query_len,
# Long BF16 prefills omit persistent metadata. gfx942's
# nonpersistent ASM path is unsupported or unsafe for these shapes.
# Sink attention has its own Triton/LSE dispatch below.
triton_prefill_fallback = (
not need_lse
and on_gfx942()
and _is_long_bf16_prefill(kv_c_and_k_pe_cache.dtype, attn_metadata)
)
if (
triton_sink_fallback
or triton_prefill_fallback
or _use_rocm_sparse_triton(
kv_cache_dtype=self.kv_cache_dtype,
head_size=q.shape[-1],
kv_lora_rank=self.kv_lora_rank,
num_prefills=attn_metadata.num_prefills,
num_decodes=attn_metadata.num_decodes,
num_decode_tokens=attn_metadata.num_decode_tokens,
max_query_len=attn_metadata.max_query_len,
)
):
output = torch.empty(
[num_tokens, q.shape[1], self.kv_lora_rank],
@@ -1078,6 +1081,7 @@ class ROCMAiterMLASparseImpl(
# Empty ragged rows have only sink mass and no value contribution.
# AITER can return NaN output/LSE for those rows; do not multiply it
# by a zero normalization factor and propagate the NaN.
assert attn_metadata.paged_kv_indptr is not None
has_keys = (
attn_metadata.paged_kv_indptr[1:] > attn_metadata.paged_kv_indptr[:-1]
).unsqueeze(-1)
@@ -1156,7 +1160,7 @@ class ROCMAiterMLASparseImpl(
else:
ops.concat_mla_q(ql_nope, q_pe, q)
num_actual_toks = attn_metadata.num_actual_tokens
num_actual_toks = q.shape[0]
# Get topk indices
assert self.topk_indices_buffer is not None
@@ -1168,7 +1172,7 @@ class ROCMAiterMLASparseImpl(
kv_c_and_k_pe_cache, attn_metadata.block_size
)
triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token,
attn_metadata.req_id_per_token[:num_actual_toks],
attn_metadata.block_table,
topk_indices,
attn_metadata.paged_kv_indptr,