[Perf] [Feat] [ROCm] Add densemha support to ROCm AITER Sparse MLA (#49263)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -2632,8 +2632,11 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/envs.py
|
||||
- vllm/platforms/rocm.py
|
||||
- vllm/models/deepseek_v32/amd/rocm.py
|
||||
commands:
|
||||
- pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- pytest -v -s v1/attention --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":amd: (MI300) Metrics, Tracing" # TBD
|
||||
timeout_in_minutes: 30
|
||||
@@ -5642,10 +5645,11 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/envs.py
|
||||
- vllm/platforms/rocm.py
|
||||
- vllm/models/deepseek_v32/amd/rocm.py
|
||||
optional: true
|
||||
commands:
|
||||
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":amd: (MI355 DPX) CUDAGraph"
|
||||
timeout_in_minutes: 55
|
||||
|
||||
@@ -30,6 +30,10 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/envs.py
|
||||
- vllm/platforms/rocm.py
|
||||
- vllm/models/deepseek_v32/amd/rocm.py
|
||||
commands:
|
||||
- pytest -v -s v1/attention --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":nvidia: (B200) V1 Attention Shard %N"
|
||||
key: v1-attention-b200
|
||||
@@ -59,9 +63,10 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/envs.py
|
||||
- vllm/platforms/rocm.py
|
||||
- vllm/models/deepseek_v32/amd/rocm.py
|
||||
commands:
|
||||
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":nvidia: (DGX) Spark FlashInfer NVFP4 XQA"
|
||||
key: flashinfer-nvfp4-xqa-dgx-spark
|
||||
|
||||
@@ -33,6 +33,101 @@ class _FakeAiter(ModuleType):
|
||||
get_mla_metadata_v1: Mock
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [16, 64, 128])
|
||||
@pytest.mark.parametrize("topk_tokens", [128, 256])
|
||||
def test_rocm_index_conversion_matrix(block_size, topk_tokens):
|
||||
"""Convert valid, masked, and out-of-range sparse indices exactly."""
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 4
|
||||
max_blocks = 4
|
||||
req_ids = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor(
|
||||
[[3, 5, 7, 9], [2, 4, 6, 8]], dtype=torch.int32, device=device
|
||||
)
|
||||
token_indices = torch.arange(
|
||||
num_tokens * topk_tokens, dtype=torch.int32, device=device
|
||||
).view(num_tokens, topk_tokens)
|
||||
token_indices %= max_blocks * block_size
|
||||
token_indices[0, 0] = -1
|
||||
token_indices[1, 1] = max_blocks * block_size
|
||||
cu_seqlens = torch.arange(
|
||||
0,
|
||||
(num_tokens + 1) * topk_tokens,
|
||||
topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
output = torch.full(
|
||||
(num_tokens * topk_tokens,), -99, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
token_indices,
|
||||
cu_seqlens,
|
||||
output,
|
||||
BLOCK_STRIDE_ROWS=block_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=topk_tokens,
|
||||
)
|
||||
|
||||
block_ids = token_indices // block_size
|
||||
valid = (token_indices >= 0) & (block_ids < max_blocks)
|
||||
safe_blocks = block_ids.clamp(0, max_blocks - 1)
|
||||
bases = block_table[req_ids[:, None].long(), safe_blocks.long()]
|
||||
expected = torch.where(valid, bases * block_size + token_indices % block_size, 0)
|
||||
torch.testing.assert_close(output.view_as(expected), expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_rocm_index_conversion_rejects_mismatched_token_extents():
|
||||
"""Reject full-batch request IDs paired with decode-subset indices."""
|
||||
with pytest.raises(AssertionError, match="must cover the same tokens"):
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
torch.zeros(3, dtype=torch.int32),
|
||||
torch.zeros(1, 4, dtype=torch.int32),
|
||||
torch.zeros(2, 128, dtype=torch.int32),
|
||||
torch.arange(3, dtype=torch.int32),
|
||||
torch.zeros(256, dtype=torch.int32),
|
||||
BLOCK_STRIDE_ROWS=64,
|
||||
NUM_TOPK_TOKENS=128,
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_index_conversion_respects_packed_valid_counts():
|
||||
"""ROCm's indptr contract packs only each token's valid top-k prefix."""
|
||||
device = torch.device("cuda")
|
||||
topk_tokens = 128
|
||||
valid_counts = torch.tensor([1, 17, 128], dtype=torch.int32, device=device)
|
||||
cu_seqlens = torch.zeros(4, dtype=torch.int32, device=device)
|
||||
torch.cumsum(valid_counts, dim=0, out=cu_seqlens[1:])
|
||||
req_ids = torch.tensor([0, 0, 0], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor([[2, 3, 4]], dtype=torch.int32, device=device)
|
||||
token_indices = torch.arange(
|
||||
3 * topk_tokens, dtype=torch.int32, device=device
|
||||
).view(3, topk_tokens)
|
||||
token_indices %= 3 * 64
|
||||
output = torch.full((int(cu_seqlens[-1]),), -99, dtype=torch.int32, device=device)
|
||||
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
token_indices,
|
||||
cu_seqlens,
|
||||
output,
|
||||
BLOCK_STRIDE_ROWS=64,
|
||||
BLOCK_SIZE=64,
|
||||
NUM_TOPK_TOKENS=topk_tokens,
|
||||
)
|
||||
|
||||
expected_parts = []
|
||||
for token_id, count in enumerate(valid_counts.tolist()):
|
||||
local = token_indices[token_id, :count]
|
||||
expected_parts.append(block_table[0, (local // 64).long()] * 64 + local % 64)
|
||||
expected = torch.cat(expected_parts)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def _make_builder():
|
||||
builder = object.__new__(sparse_mod.ROCMAiterMLASparseMetadataBuilder)
|
||||
max_num_batched_tokens = 8
|
||||
@@ -44,6 +139,18 @@ def _make_builder():
|
||||
builder.kv_cache_dtype = "fp8"
|
||||
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
|
||||
builder.topk_tokens = topk_tokens
|
||||
# SparseMLACommonMetadataBuilder state consumed by its build() method.
|
||||
builder.cp_kv_cache_interleave_size = 1
|
||||
builder.use_pcp = False
|
||||
builder._prefill_backend = None
|
||||
|
||||
def cpu_token_to_req(metadata):
|
||||
return torch.repeat_interleave(
|
||||
torch.arange(metadata.num_reqs, dtype=torch.int32),
|
||||
metadata.query_start_loc_cpu.diff(),
|
||||
)
|
||||
|
||||
builder._build_req_id_per_token = cpu_token_to_req # type: ignore[method-assign]
|
||||
builder.req_id_per_token_buffer = torch.zeros(
|
||||
max_num_batched_tokens, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
@@ -62,6 +169,8 @@ def _make_builder():
|
||||
builder._use_persistent_metadata = True
|
||||
builder._num_attention_heads = 16
|
||||
builder._num_compute_units = current_platform.num_compute_units()
|
||||
builder._mla_q_dtype = torch.bfloat16
|
||||
builder._mla_kv_dtype = torch.bfloat16
|
||||
builder._mla_work_meta_data = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_work_indptr = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_work_info_set = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
@@ -216,6 +325,8 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
|
||||
assert events == ["metadata", "sync"]
|
||||
assert fake_get_mla_metadata_v1_mock.call_count == 1
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["max_split_per_batch"] == 1
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_q"] == torch.bfloat16
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_kv"] == torch.bfloat16
|
||||
|
||||
events.clear()
|
||||
|
||||
@@ -223,3 +334,395 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
|
||||
|
||||
assert events == []
|
||||
assert fake_get_mla_metadata_v1_mock.call_count == 1
|
||||
|
||||
|
||||
def test_dense_prefill_builds_sparse_metadata_for_decode_subset(monkeypatch):
|
||||
"""Dense prefill must exclude prefill tokens from AITER sparse metadata.
|
||||
|
||||
A mixed batch is routed as sparse MQA for its leading decode requests and
|
||||
dense MHA for its trailing short prefill requests. The AITER metadata
|
||||
buffers therefore must describe only the decode subset; including prefill
|
||||
tokens would mismatch the query passed to ``forward_mqa``.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
|
||||
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([16, 4], dtype=torch.int32)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=4,
|
||||
max_seq_len=16,
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=4,
|
||||
max_seq_len=16,
|
||||
num_actual_tokens=5,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(5),
|
||||
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
req_id_per_token=torch.tensor([0, 1, 1, 1, 1], dtype=torch.int32),
|
||||
topk_tokens=4,
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=4,
|
||||
prefill=SimpleNamespace(use_dense_mha=True),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
sparse_call: dict[str, torch.Tensor] = {}
|
||||
|
||||
def fake_generate_sparse_seqlen(
|
||||
query_lens, sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
|
||||
):
|
||||
sparse_call.update(
|
||||
query_lens=query_lens.clone(),
|
||||
seq_lens=sparse_seq_lens.clone(),
|
||||
query_start_loc=sparse_query_start_loc.clone(),
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
return torch.ones(num_tokens, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
|
||||
)
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
fake_aiter.get_mla_metadata_v1 = Mock()
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
assert result.prefill is metadata.prefill
|
||||
assert result.num_prefills == 1
|
||||
assert result.req_id_per_token.tolist() == [0]
|
||||
assert result.qo_indptr.numel() == 2
|
||||
assert result.paged_kv_indptr.numel() == 2
|
||||
assert sparse_call["num_tokens"] == 1
|
||||
assert sparse_call["query_lens"].tolist() == [1]
|
||||
assert sparse_call["seq_lens"].tolist() == [16]
|
||||
assert sparse_call["query_start_loc"].tolist() == [0, 1]
|
||||
|
||||
|
||||
def test_long_prefill_shares_full_sparse_metadata_with_decode(monkeypatch):
|
||||
"""A prefill longer than top-k must remain in shared sparse metadata.
|
||||
|
||||
Dense MHA is only selected when the prefill sequence fits within the
|
||||
sparse top-k window. Once it exceeds that window, both decode and prefill
|
||||
are handled by sparse MQA, so every query token must be represented by the
|
||||
same sparse paging metadata.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
|
||||
# One decode token followed by a five-token prefill. topk_tokens is four,
|
||||
# so the prefill must not be split onto the dense-MHA path.
|
||||
query_start_loc = torch.tensor([0, 1, 6], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([16, 5], dtype=torch.int32)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=5,
|
||||
max_seq_len=16,
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=5,
|
||||
max_seq_len=16,
|
||||
num_actual_tokens=6,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(6),
|
||||
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
req_id_per_token=torch.tensor([0, 1, 1, 1, 1, 1], dtype=torch.int32),
|
||||
topk_tokens=4,
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=5,
|
||||
prefill=SimpleNamespace(use_dense_mha=False),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
sparse_call: dict[str, torch.Tensor] = {}
|
||||
|
||||
def fake_generate_sparse_seqlen(
|
||||
query_lens, sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
|
||||
):
|
||||
sparse_call.update(
|
||||
query_lens=query_lens.clone(),
|
||||
seq_lens=sparse_seq_lens.clone(),
|
||||
query_start_loc=sparse_query_start_loc.clone(),
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
return torch.ones(num_tokens, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
|
||||
)
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
get_persistent_metadata = Mock()
|
||||
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
assert result.num_decodes == 1
|
||||
assert result.num_prefills == 1
|
||||
assert result.req_id_per_token.tolist() == [0, 1, 1, 1, 1, 1]
|
||||
assert result.qo_indptr.numel() == 7
|
||||
assert result.paged_kv_indptr.numel() == 7
|
||||
assert result.paged_kv_indices.numel() == 6 * result.topk_tokens
|
||||
assert sparse_call["num_tokens"] == 6
|
||||
assert sparse_call["query_lens"].tolist() == [1, 5]
|
||||
assert sparse_call["seq_lens"].tolist() == [16, 5]
|
||||
assert sparse_call["query_start_loc"].tolist() == [0, 1, 6]
|
||||
get_persistent_metadata.assert_called_once()
|
||||
assert result.work_meta_data is builder._mla_work_meta_data
|
||||
assert result.reduce_indptr is builder._mla_reduce_indptr
|
||||
|
||||
|
||||
def test_forward_mqa_slices_full_batch_metadata_to_query_tokens(monkeypatch):
|
||||
"""The sparse kernel inputs must have the same token extent as its query.
|
||||
|
||||
In a mixed dense-prefill batch, ``forward_mqa`` receives only decode query
|
||||
tokens while some metadata buffers may represent the full batch. This
|
||||
guards the slicing that prevents the index conversion from writing beyond
|
||||
the decode-sized output.
|
||||
"""
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.num_heads = 4
|
||||
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
|
||||
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
|
||||
forward_mla = Mock(return_value=(expected_output, None))
|
||||
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
|
||||
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
|
||||
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
|
||||
block_size=1,
|
||||
topk_tokens=4,
|
||||
)
|
||||
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
|
||||
captured = {}
|
||||
|
||||
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
|
||||
captured["req_ids"] = req_ids.clone()
|
||||
captured["token_indices"] = token_indices.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
|
||||
)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
torch.zeros(1, 1, 576, dtype=torch.bfloat16),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert output is expected_output
|
||||
assert lse is None
|
||||
assert captured["req_ids"].tolist() == [0, 1]
|
||||
assert captured["token_indices"].shape == (2, 4)
|
||||
forward_mla.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_forward_mqa_slices_decode_subset_for_bf16_and_fp8(monkeypatch, kv_cache_dtype):
|
||||
"""Both cache dtypes must slice mixed-batch metadata to decode queries."""
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.kv_cache_dtype = kv_cache_dtype
|
||||
impl.num_heads = 4
|
||||
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
|
||||
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
|
||||
forward_mla = Mock(return_value=(expected_output, None))
|
||||
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
|
||||
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
|
||||
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
|
||||
block_size=1,
|
||||
topk_tokens=4,
|
||||
)
|
||||
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.zeros(1, 1, 576, dtype=torch.uint8)
|
||||
captured = {}
|
||||
|
||||
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
|
||||
captured["req_ids"] = req_ids.clone()
|
||||
captured["token_indices"] = token_indices.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
|
||||
)
|
||||
if kv_cache_dtype == "fp8":
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.ops,
|
||||
"scaled_fp8_quant",
|
||||
lambda q, scale: (q.to(fp8_dtype), None),
|
||||
)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
kv_cache,
|
||||
metadata,
|
||||
SimpleNamespace(_q_scale=torch.tensor(1.0)),
|
||||
)
|
||||
|
||||
assert output is expected_output
|
||||
assert lse is None
|
||||
assert captured["req_ids"].tolist() == [0, 1]
|
||||
assert captured["token_indices"].shape == (2, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("length_relation", ["short", "long"])
|
||||
@pytest.mark.parametrize("workload", ["prefill", "mixed", "decode"])
|
||||
def test_sparse_metadata_workload_length_dtype_matrix(
|
||||
monkeypatch, workload, length_relation, kv_cache_dtype
|
||||
):
|
||||
"""Cover routing and persistent metadata across the ROCm support matrix.
|
||||
|
||||
The matrix crosses pure prefill, mixed prefill/decode, and decode-only
|
||||
workloads with sequence lengths below/above top-k and BF16/FP8 KV cache.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
if kv_cache_dtype == "fp8":
|
||||
builder._mla_q_dtype = current_platform.fp8_dtype()
|
||||
builder._mla_kv_dtype = current_platform.fp8_dtype()
|
||||
|
||||
is_long = length_relation == "long"
|
||||
prefill_len = 5 if is_long else 3
|
||||
decode_seq_len = 8 if is_long else 3
|
||||
if workload == "prefill":
|
||||
query_lens = [prefill_len]
|
||||
seq_lens_list = [prefill_len]
|
||||
num_decodes, num_prefills = 0, 1
|
||||
elif workload == "mixed":
|
||||
query_lens = [1, prefill_len]
|
||||
seq_lens_list = [decode_seq_len, prefill_len]
|
||||
num_decodes, num_prefills = 1, 1
|
||||
else:
|
||||
query_lens = [1, 1]
|
||||
seq_lens_list = [decode_seq_len, decode_seq_len + 1]
|
||||
num_decodes, num_prefills = 2, 0
|
||||
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
)
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32)
|
||||
num_tokens = sum(query_lens)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=max(query_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=len(query_lens),
|
||||
max_query_len=max(query_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(num_tokens),
|
||||
block_table=torch.arange(len(query_lens) * 8, dtype=torch.int32).view(
|
||||
len(query_lens), 8
|
||||
),
|
||||
req_id_per_token=torch.repeat_interleave(
|
||||
torch.arange(len(query_lens), dtype=torch.int32),
|
||||
torch.tensor(query_lens),
|
||||
),
|
||||
topk_tokens=4,
|
||||
num_decodes=num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decodes,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=prefill_len if num_prefills else 0,
|
||||
prefill=SimpleNamespace(use_dense_mha=not is_long) if num_prefills else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"generate_sparse_seqlen_triton",
|
||||
lambda *args, **kwargs: torch.ones(args[4], dtype=torch.int32),
|
||||
)
|
||||
get_persistent_metadata = Mock()
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
uses_dense_prefill = workload in ("prefill", "mixed") and not is_long
|
||||
expected_sparse_tokens = num_decodes if uses_dense_prefill else num_tokens
|
||||
assert result.req_id_per_token.numel() == expected_sparse_tokens
|
||||
assert result.qo_indptr.numel() == expected_sparse_tokens + 1
|
||||
assert result.paged_kv_indptr.numel() == expected_sparse_tokens + 1
|
||||
|
||||
uses_fallback = workload == "prefill" and is_long and kv_cache_dtype == "auto"
|
||||
if expected_sparse_tokens == 0 or uses_fallback:
|
||||
get_persistent_metadata.assert_not_called()
|
||||
else:
|
||||
get_persistent_metadata.assert_called_once()
|
||||
if uses_fallback or expected_sparse_tokens == 0:
|
||||
assert result.work_meta_data is None
|
||||
assert result.reduce_indptr is None
|
||||
else:
|
||||
assert result.work_meta_data is builder._mla_work_meta_data
|
||||
assert result.reduce_indptr is builder._mla_reduce_indptr
|
||||
|
||||
@@ -155,8 +155,86 @@ def test_rocm_sparse_triton_route(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("gfx942", "fp8", "num_prefills", "num_decodes", "prefill_max_seq_len", "route"),
|
||||
[
|
||||
pytest.param(True, False, 1, 0, 129, "triton", id="above-topk"),
|
||||
pytest.param(False, False, 1, 0, 129, "aiter", id="gfx950"),
|
||||
pytest.param(True, True, 1, 0, 129, "aiter", id="fp8"),
|
||||
pytest.param(True, False, 0, 2, 129, "aiter", id="decode-only"),
|
||||
pytest.param(True, False, 1, 1, 129, "aiter", id="mixed"),
|
||||
pytest.param(True, False, 1, 0, 128, "aiter", id="equal-topk"),
|
||||
pytest.param(True, False, 1, 0, 127, "aiter", id="below-topk"),
|
||||
pytest.param(True, False, 0, 0, 129, "aiter", id="empty"),
|
||||
pytest.param(
|
||||
True, False, 1, 0, None, "missing", id="required-prefill-metadata"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_rocm_long_bf16_prefill_dispatch(
|
||||
monkeypatch, gfx942, fp8, num_prefills, num_decodes, prefill_max_seq_len, route
|
||||
):
|
||||
"""Check the new dispatch gate using CPU tensors and intercepted kernels."""
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942)
|
||||
|
||||
class KernelSelected(Exception):
|
||||
pass
|
||||
|
||||
def triton_kernel(**kwargs):
|
||||
raise KernelSelected("triton")
|
||||
|
||||
def aiter_kernel(*args, **kwargs):
|
||||
raise KernelSelected("aiter")
|
||||
|
||||
monkeypatch.setattr(sparse_mod, "rocm_sparse_attn_prefill", triton_kernel)
|
||||
monkeypatch.setattr(sparse_mod.rocm_aiter_ops, "mla_decode_fwd", aiter_kernel)
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = 64
|
||||
impl.kv_lora_rank = 512
|
||||
impl.kv_cache_dtype = "fp8" if fp8 else "auto"
|
||||
impl.scale = 576**-0.5
|
||||
impl.sinks = None
|
||||
|
||||
dtype = torch.float8_e4m3fn if fp8 else torch.bfloat16
|
||||
q = torch.empty(2, 64, 576, dtype=dtype)
|
||||
kv = torch.empty(4, 1, 576, dtype=dtype)
|
||||
metadata = SimpleNamespace(
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
num_prefills=num_prefills,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decodes,
|
||||
max_query_len=2,
|
||||
prefill_max_seq_len=prefill_max_seq_len,
|
||||
topk_tokens=128,
|
||||
work_meta_data=None,
|
||||
qo_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.empty(0, dtype=torch.int32),
|
||||
paged_kv_indptr=torch.zeros(3, dtype=torch.int32),
|
||||
paged_kv_last_page_len=torch.ones(2, dtype=torch.int32),
|
||||
)
|
||||
# Irrelevant routes must not inspect prefill-only metadata.
|
||||
if (
|
||||
not gfx942
|
||||
or fp8
|
||||
or num_decodes
|
||||
or not num_prefills
|
||||
or prefill_max_seq_len is None
|
||||
):
|
||||
del metadata.prefill_max_seq_len
|
||||
del metadata.topk_tokens
|
||||
layer = SimpleNamespace(_q_scale=1.0, _k_scale=1.0)
|
||||
error = AttributeError if route == "missing" else KernelSelected
|
||||
match = "prefill_max_seq_len" if route == "missing" else f"^{route}$"
|
||||
with pytest.raises(error, match=match):
|
||||
impl._forward_mla(layer, q, kv, metadata)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gfx942", [False, True])
|
||||
@pytest.mark.parametrize("num_heads", [8, 12])
|
||||
def test_rocm_sparse_triton_route_preserves_padded_sinks(monkeypatch, num_heads):
|
||||
def test_rocm_sparse_triton_route_preserves_padded_sinks(
|
||||
monkeypatch, num_heads, gfx942
|
||||
):
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942)
|
||||
captured = {}
|
||||
|
||||
def fake_rocm_sparse_attn_prefill(**kwargs):
|
||||
|
||||
@@ -47,13 +47,12 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# TODO: Integrate ROCMAiterMLASparseBackend for ROCm.
|
||||
# The ROCm sparse MLA backend (rocm_aiter_mla_sparse.py) has a compatible
|
||||
# forward_mqa interface but needs validation on ROCm hardware.
|
||||
if not current_platform.is_cuda():
|
||||
# This module's end-to-end sparse-backend correctness test supports CUDA and
|
||||
# ROCm. The remaining tests exercise CUDA-specific FlashMLA/FlashInfer APIs
|
||||
# and are skipped by the autouse fixture below when collecting on ROCm.
|
||||
if not (current_platform.is_cuda() or current_platform.is_rocm()):
|
||||
pytest.skip(
|
||||
"Sparse MLA backend tests currently only support CUDA. "
|
||||
"ROCm support requires integrating ROCMAiterMLASparseBackend.",
|
||||
"Sparse MLA backend tests require CUDA or ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
@@ -65,14 +64,16 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.attention.backends.mla import (
|
||||
flashattn_mla_sparse as flashattn_sparse_module,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla import index_group as index_group_module
|
||||
from vllm.v1.attention.backends.mla.flashattn_mla_sparse import (
|
||||
FlashAttnMLASparseImpl,
|
||||
FlashAttnMLASparseMetadataBuilder,
|
||||
)
|
||||
|
||||
if current_platform.is_cuda():
|
||||
from vllm.v1.attention.backends.mla import (
|
||||
flashattn_mla_sparse as flashattn_sparse_module,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashattn_mla_sparse import (
|
||||
FlashAttnMLASparseImpl,
|
||||
FlashAttnMLASparseMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseImpl,
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
@@ -111,9 +112,43 @@ from vllm.v1.hisparse.runtime import (
|
||||
)
|
||||
from vllm.v1.hisparse.types import SparseKVRowMirror
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
|
||||
SPARSE_BACKENDS = [ROCMAiterMLASparseBackend]
|
||||
SPARSE_BACKEND_IDS = ["ROCMAiterMLA"]
|
||||
SPARSE_KV_CACHE_DTYPES = ["auto", "fp8"]
|
||||
SPARSE_BLOCK_SIZES = [64]
|
||||
else:
|
||||
SPARSE_BACKENDS = [FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend]
|
||||
SPARSE_BACKEND_IDS = ["FlashMLA", "FlashInferTRTLLM"]
|
||||
SPARSE_KV_CACHE_DTYPES = ["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"]
|
||||
SPARSE_BLOCK_SIZES = [32, 64]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _skip_cuda_specific_sparse_mla_tests_on_rocm(request):
|
||||
"""Skip tests whose helpers exercise CUDA-only sparse MLA APIs."""
|
||||
rocm_portable_tests = {
|
||||
"test_sparse_backend_decode_correctness",
|
||||
"test_sparse_backend_prefill_correctness",
|
||||
"test_rocm_fused_v32_dense_prefill_matches_mqa",
|
||||
"test_rocm_long_bf16_prefill_avoids_nonpersistent_aiter",
|
||||
}
|
||||
if (
|
||||
current_platform.is_rocm()
|
||||
and getattr(request.node, "originalname", request.node.name)
|
||||
not in rocm_portable_tests
|
||||
):
|
||||
pytest.skip("This sparse MLA test exercises a CUDA-specific API")
|
||||
|
||||
|
||||
SPARSE_BACKEND_BATCH_SPECS = {
|
||||
name: BATCH_SPECS[name]
|
||||
for name in [
|
||||
"small_decode",
|
||||
"mixed_small",
|
||||
"mixed_medium",
|
||||
"small_prefill",
|
||||
@@ -128,10 +163,61 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_prefill"] = BatchSpec(
|
||||
SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
|
||||
seq_lens=[256] * 2, query_lens=[256] * 2
|
||||
)
|
||||
SPARSE_BACKEND_BATCH_SPECS["large_decode_over_topk"] = BatchSpec(
|
||||
seq_lens=[256, 384], query_lens=[1, 1]
|
||||
)
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch_name", "tensor_parallel_size"),
|
||||
[
|
||||
("large_q_prefill", 2),
|
||||
("large_q_pure_prefill", 4),
|
||||
("large_q_pure_prefill", 1),
|
||||
],
|
||||
)
|
||||
def test_rocm_long_bf16_prefill_avoids_nonpersistent_aiter(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
monkeypatch,
|
||||
batch_name,
|
||||
tensor_parallel_size,
|
||||
):
|
||||
"""MI300 long BF16 prefills must remain correct without AITER ASM dispatch."""
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm sparse MLA regression")
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# Exercise the gfx942 routing decision on other ROCm devices too;
|
||||
# the numerical kernel still compiles for the actual device.
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: True)
|
||||
|
||||
def unsupported_aiter(*args, **kwargs):
|
||||
pytest.fail("Long gfx942 BF16 prefill reached nonpersistent AITER")
|
||||
|
||||
monkeypatch.setattr(rocm_aiter_ops, "mla_decode_fwd", unsupported_aiter)
|
||||
test_sparse_backend_decode_correctness(
|
||||
default_vllm_config=default_vllm_config,
|
||||
dist_init=dist_init,
|
||||
backend_cls=ROCMAiterMLASparseBackend,
|
||||
batch_name=batch_name,
|
||||
kv_cache_dtype="auto",
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
block_size=64,
|
||||
workspace_init=workspace_init,
|
||||
q_scale=1.0,
|
||||
k_scale=1.0,
|
||||
total_num_heads=128,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
|
||||
|
||||
def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
"""Weight absorption must not change the model's attention temperature."""
|
||||
model_scale = 256**-0.5
|
||||
@@ -403,16 +489,13 @@ def _quantize_dequantize_nvfp4_ds_mla(
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_cls",
|
||||
[FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend],
|
||||
ids=["FlashMLA", "FlashInferTRTLLM"],
|
||||
SPARSE_BACKENDS,
|
||||
ids=SPARSE_BACKEND_IDS,
|
||||
)
|
||||
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype",
|
||||
["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"],
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", SPARSE_KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
||||
@pytest.mark.parametrize("block_size", [32, 64])
|
||||
@pytest.mark.parametrize("block_size", SPARSE_BLOCK_SIZES)
|
||||
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
|
||||
@pytest.mark.parametrize(
|
||||
("total_num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
|
||||
@@ -525,6 +608,12 @@ def test_sparse_backend_decode_correctness(
|
||||
"attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
|
||||
},
|
||||
)
|
||||
# This shared correctness test constructs a sparse SDPA reference for every
|
||||
# query token. ROCm normally routes short prefills to dense MHA, whose
|
||||
# output intentionally differs from that sparse reference. Keep this test
|
||||
# on the sparse-MQA path; dense-prefill routing has dedicated ROCm tests.
|
||||
if current_platform.is_rocm():
|
||||
vllm_config.attention_config.sparse_mla_force_mqa = True
|
||||
model_config = vllm_config.model_config
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
index_topk=topk_tokens,
|
||||
@@ -737,7 +826,7 @@ def test_sparse_backend_decode_correctness(
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
vllm_config.compilation_config.static_forward_context["placeholder"] = (
|
||||
SimpleNamespace(prefill_backend=prefill_backend)
|
||||
SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace())
|
||||
)
|
||||
|
||||
builder_cls = backend_cls.get_builder_cls()
|
||||
@@ -1384,11 +1473,13 @@ PREFILL_BATCH_SPECS = {
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability()[0] < 10,
|
||||
current_platform.is_cuda() and torch.cuda.get_device_capability()[0] < 10,
|
||||
reason="Sparse MLA forward_mha requires FA4 (SM100+)",
|
||||
)
|
||||
@pytest.mark.parametrize("batch_name", list(PREFILL_BATCH_SPECS.keys()))
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype", (["auto", "fp8"] if current_platform.is_rocm() else ["auto"])
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
|
||||
[
|
||||
@@ -1408,11 +1499,19 @@ def test_sparse_backend_prefill_correctness(
|
||||
v_head_dim,
|
||||
workspace_init,
|
||||
):
|
||||
"""Test dense and masked MHA across supported sparse MLA dimensions."""
|
||||
backend_cls = FlashMLASparseBackend
|
||||
"""Test the normal dense forward_mha path for sparse MLA short prefill."""
|
||||
backend_cls = (
|
||||
ROCMAiterMLASparseBackend
|
||||
if current_platform.is_rocm()
|
||||
else FlashMLASparseBackend
|
||||
)
|
||||
if current_platform.is_rocm() and (
|
||||
batch_name.startswith("masked_mha") or qk_nope_head_dim != 128
|
||||
):
|
||||
pytest.skip("ROCm dense prefill requires the DeepSeek MLA layout")
|
||||
batch_spec = PREFILL_BATCH_SPECS[batch_name]
|
||||
|
||||
device = torch.device("cuda")
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
dtype = torch.bfloat16
|
||||
block_size = 64
|
||||
|
||||
@@ -1579,7 +1678,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
vllm_config.compilation_config.static_forward_context["placeholder"] = (
|
||||
SimpleNamespace(prefill_backend=prefill_backend)
|
||||
SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace())
|
||||
)
|
||||
|
||||
builder_cls = backend_cls.get_builder_cls()
|
||||
@@ -1653,7 +1752,10 @@ def test_sparse_backend_prefill_correctness(
|
||||
|
||||
assert out_buffer.shape == ref_output.shape
|
||||
assert torch.isfinite(out_buffer).all(), "Non-finite values in output"
|
||||
torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01)
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
torch.testing.assert_close(out_buffer, ref_output, rtol=0.065, atol=0.05)
|
||||
else:
|
||||
torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -3057,7 +3159,7 @@ def test_hisparse_mixed_batch_bf16_row_split(
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
vllm_config.compilation_config.static_forward_context["placeholder"] = (
|
||||
SimpleNamespace(prefill_backend=prefill_backend)
|
||||
SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace())
|
||||
)
|
||||
|
||||
builder_cls = FlashMLASparseBackend.get_builder_cls()
|
||||
@@ -4264,3 +4366,118 @@ def test_sparse_mla_common_impl_resolves_buffer_lazily():
|
||||
assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
|
||||
"topk_indices_buffer must stay a lazily-resolved property"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm model implementation")
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_rocm_fused_v32_dense_prefill_matches_mqa(
|
||||
tmp_path, vllm_runner, kv_cache_dtype
|
||||
):
|
||||
"""Fused normalization, RoPE and mixed dispatch preserve generation/logprobs.
|
||||
|
||||
Chunking unequal prompts exercises prefill alongside decode. The two long
|
||||
prompts straddle the dense-prefill threshold. No pretrained weights or
|
||||
tokenizer download is needed.
|
||||
"""
|
||||
import json
|
||||
|
||||
from vllm import SamplingParams
|
||||
|
||||
config = {
|
||||
"architectures": ["DeepseekV32ForCausalLM"],
|
||||
"attention_bias": False,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 0,
|
||||
"eos_token_id": 1,
|
||||
"ep_size": 1,
|
||||
"first_k_dense_replace": 1,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 2048,
|
||||
"index_head_dim": 128,
|
||||
"index_n_heads": 64,
|
||||
"index_topk": 2048,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4096,
|
||||
"kv_lora_rank": 512,
|
||||
"max_position_embeddings": 8192,
|
||||
"model_type": "deepseek_v32",
|
||||
"moe_intermediate_size": 512,
|
||||
"moe_layer_freq": 1,
|
||||
"n_group": 1,
|
||||
"n_routed_experts": 8,
|
||||
"n_shared_experts": 1,
|
||||
"norm_topk_prob": True,
|
||||
"num_attention_heads": 32,
|
||||
"num_experts_per_tok": 2,
|
||||
"num_hidden_layers": 2,
|
||||
"num_key_value_heads": 32,
|
||||
"num_nextn_predict_layers": 0,
|
||||
"q_lora_rank": 512,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": {
|
||||
"beta_fast": 32,
|
||||
"beta_slow": 1,
|
||||
"factor": 40,
|
||||
"mscale": 1.0,
|
||||
"mscale_all_dim": 1.0,
|
||||
"original_max_position_embeddings": 4096,
|
||||
"type": "yarn",
|
||||
},
|
||||
"rope_theta": 10000,
|
||||
"routed_scaling_factor": 2.5,
|
||||
"scoring_func": "sigmoid",
|
||||
"tie_word_embeddings": False,
|
||||
"topk_group": 1,
|
||||
"topk_method": "noaux_tc",
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.44.2",
|
||||
"use_cache": True,
|
||||
"v_head_dim": 128,
|
||||
"vocab_size": 4096,
|
||||
}
|
||||
(tmp_path / "config.json").write_text(json.dumps(config))
|
||||
prompts = [
|
||||
{"prompt_token_ids": [2 + (i * 17 + j * 13) % 4000 for i in range(n)]}
|
||||
for j, n in enumerate([64, 128, 2048, 2049])
|
||||
]
|
||||
sampling = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True, logprobs=5)
|
||||
results = []
|
||||
for force_mqa in [True, False]:
|
||||
with vllm_runner(
|
||||
str(tmp_path),
|
||||
load_format="dummy",
|
||||
skip_tokenizer_init=True,
|
||||
dtype="bfloat16",
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
seed=42,
|
||||
max_model_len=4096,
|
||||
max_num_seqs=4,
|
||||
max_num_batched_tokens=256,
|
||||
block_size=64,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=False,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.25,
|
||||
attention_config={
|
||||
"backend": "ROCM_AITER_MLA_SPARSE",
|
||||
"sparse_mla_force_mqa": force_mqa,
|
||||
},
|
||||
) as runner:
|
||||
outputs = runner.llm.generate(prompts, sampling, use_tqdm=False)
|
||||
results.append([request.outputs[0] for request in outputs])
|
||||
|
||||
for reference, actual in zip(*results):
|
||||
assert len(actual.token_ids) == 16
|
||||
assert actual.token_ids == reference.token_ids
|
||||
assert actual.logprobs is not None and reference.logprobs is not None
|
||||
for token, expected, observed in zip(
|
||||
actual.token_ids, reference.logprobs, actual.logprobs
|
||||
):
|
||||
torch.testing.assert_close(
|
||||
torch.tensor(observed[token].logprob),
|
||||
torch.tensor(expected[token].logprob),
|
||||
rtol=0.01,
|
||||
atol=0.01,
|
||||
)
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer
|
||||
@@ -179,6 +182,8 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
else:
|
||||
attn_metadata = attn_metadata_raw
|
||||
|
||||
attn_metadata = cast(MLACommonMetadata | None, attn_metadata)
|
||||
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping, dict)
|
||||
mla_slot = slot_mapping.get(self.layer_name)
|
||||
@@ -213,6 +218,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
mla_kv_cache = self.kv_cache
|
||||
mla_k_scale = self._k_scale
|
||||
|
||||
use_mha = attn_metadata is not None and self._use_sparse_mha(attn_metadata)
|
||||
kv_c_out = torch.empty_like(kv_c) if use_mha else None
|
||||
k_pe_out = torch.empty_like(k_pe) if use_mha else None
|
||||
|
||||
q_c = fused_norm_rope(
|
||||
positions,
|
||||
q_c,
|
||||
@@ -237,9 +246,18 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
mla_k_scale=mla_k_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
kv_c_out=kv_c_out,
|
||||
k_pe_out=k_pe_out,
|
||||
)
|
||||
|
||||
ql_nope, q_pe = self._compute_ql_nope(q_c)
|
||||
if use_mha:
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split(
|
||||
[self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
ql_nope = q_nope
|
||||
else:
|
||||
ql_nope, q_pe = self._compute_ql_nope(q_c)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
index_q = self.indexer.wq_b(q_c)[0]
|
||||
@@ -260,7 +278,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
indexer_n_head_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
quantize_mqa=self._fp8_kv,
|
||||
quantize_mqa=self._fp8_kv and not use_mha,
|
||||
)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
@@ -270,6 +288,19 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
if use_mha:
|
||||
assert kv_c_out is not None and k_pe_out is not None
|
||||
mha_q = torch.cat((q_nope, mqa_q), dim=-1)
|
||||
self.forward_impl(
|
||||
mha_q,
|
||||
kv_c_out,
|
||||
k_pe_out.unsqueeze(1),
|
||||
self.kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
)
|
||||
return
|
||||
|
||||
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
|
||||
kv_cache = self.kv_cache
|
||||
if self._fp8_kv_needs_view:
|
||||
|
||||
@@ -12,11 +12,10 @@ from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
get_mla_dims,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
SharedTopkIndicesBuffer,
|
||||
SparseMLACommonImpl,
|
||||
SparseMLACommonMetadata,
|
||||
SparseMLACommonMetadataBuilder,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -26,10 +25,7 @@ from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionLayer,
|
||||
AttentionMetadata,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
MLAAttentionImpl,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
@@ -39,7 +35,6 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
align_blocks_to_rows,
|
||||
flat_kv_row_view,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_sparse_attn_decode_bf16,
|
||||
rocm_sparse_attn_prefill,
|
||||
@@ -76,6 +71,18 @@ def _use_rocm_sparse_triton(
|
||||
)
|
||||
|
||||
|
||||
def _is_long_bf16_prefill(
|
||||
kv_dtype: torch.dtype,
|
||||
metadata: "ROCMAiterMLASparseMetadata",
|
||||
) -> bool:
|
||||
return (
|
||||
kv_dtype == torch.bfloat16
|
||||
and metadata.num_decodes == 0
|
||||
and metadata.num_prefills > 0
|
||||
and metadata.prefill_max_seq_len > metadata.topk_tokens
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fit_kpool_indices_kernel(
|
||||
token_indices_ptr, # int32 [num_tokens, NUM_TOPK_TOKENS + TAIL_WIDTH]
|
||||
@@ -239,6 +246,10 @@ def triton_convert_req_index_to_global_index(
|
||||
assert req_id.dtype == torch.int32
|
||||
assert block_table.dtype == torch.int32
|
||||
assert token_indices.dtype == torch.int32
|
||||
assert req_id.shape[0] == token_indices.shape[0], (
|
||||
f"req_id ({req_id.shape[0]}) and token_indices ({token_indices.shape[0]}) "
|
||||
"must cover the same tokens"
|
||||
)
|
||||
assert token_indices.shape[1] == NUM_TOPK_TOKENS
|
||||
assert NUM_TOPK_TOKENS % BLOCK_N == 0, (
|
||||
f"NUM_TOPK_TOKENS ({NUM_TOPK_TOKENS}) must be divisible byBLOCK_N ({BLOCK_N})"
|
||||
@@ -390,35 +401,13 @@ class ROCMAiterMLASparseBackend(AttentionBackend):
|
||||
return on_mi3xx()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ROCMAiterMLASparseMetadata(AttentionMetadata):
|
||||
num_reqs: int
|
||||
max_query_len: int
|
||||
max_seq_len: int
|
||||
|
||||
num_actual_tokens: int # Number of tokens excluding padding.
|
||||
query_start_loc: torch.Tensor
|
||||
slot_mapping: torch.Tensor
|
||||
|
||||
block_table: torch.Tensor
|
||||
req_id_per_token: torch.Tensor
|
||||
|
||||
qo_indptr: torch.Tensor
|
||||
paged_kv_last_page_len: torch.Tensor
|
||||
paged_kv_indices: torch.Tensor
|
||||
paged_kv_indptr: torch.Tensor
|
||||
attn_out_dtype: torch.dtype
|
||||
|
||||
block_size: int = 1
|
||||
topk_tokens: int = 2048
|
||||
|
||||
# Fields read by the shared MLA forward. This impl has no dense-MHA prefill
|
||||
# path (supports_dense_mha_prefill=False), so it always runs the MQA path;
|
||||
num_decodes: int = 0
|
||||
num_prefills: int = 0
|
||||
num_decode_tokens: int = 0
|
||||
prefill_max_seq_len: int = 0
|
||||
prefill: object = None
|
||||
@dataclass(kw_only=True)
|
||||
class ROCMAiterMLASparseMetadata(SparseMLACommonMetadata):
|
||||
qo_indptr: torch.Tensor | None = None
|
||||
paged_kv_last_page_len: torch.Tensor | None = None
|
||||
paged_kv_indices: torch.Tensor | None = None
|
||||
paged_kv_indptr: torch.Tensor | None = None
|
||||
attn_out_dtype: torch.dtype | None = None
|
||||
|
||||
# Persistent MLA metadata (only populated when persistent mode is enabled,
|
||||
# i.e. when the aiter sparse decode kernel supports work-stealing splits).
|
||||
@@ -432,9 +421,10 @@ class ROCMAiterMLASparseMetadata(AttentionMetadata):
|
||||
|
||||
@dataclass
|
||||
class ROCMAiterMLASparseMetadataBuilder(
|
||||
AttentionMetadataBuilder[ROCMAiterMLASparseMetadata]
|
||||
SparseMLACommonMetadataBuilder[ROCMAiterMLASparseMetadata]
|
||||
):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
metadata_cls = ROCMAiterMLASparseMetadata
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -443,20 +433,15 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
self.model_config = vllm_config.model_config
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.model_dtype = vllm_config.model_config.dtype
|
||||
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.device = device
|
||||
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
||||
|
||||
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
self.topk_tokens = vllm_config.model_config.hf_text_config.index_topk
|
||||
attention_context = vllm_config.compilation_config.static_forward_context
|
||||
# Sink decode must use AITER's nonpersistent path. In particular,
|
||||
# gfx942 has no persistent+LSE kernel, and its metadata heuristic
|
||||
@@ -521,9 +506,12 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
|
||||
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
|
||||
kv_cache_dtype_str = "fp8"
|
||||
q_dtype = dtypes.fp8
|
||||
else:
|
||||
kv_cache_dtype_str = "bf16"
|
||||
kv_dtype = dtypes.d_dtypes.get(kv_cache_dtype_str, dtypes.bf16)
|
||||
self._mla_q_dtype = q_dtype
|
||||
self._mla_kv_dtype = kv_dtype
|
||||
|
||||
(
|
||||
(work_meta_data_size, work_meta_data_type),
|
||||
@@ -587,13 +575,19 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> ROCMAiterMLASparseMetadata:
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
(num_decodes, num_prefills, num_decode_tokens, _) = split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold or 1,
|
||||
metadata = super().build(
|
||||
common_prefix_len, common_attn_metadata, fast_build=fast_build
|
||||
)
|
||||
use_dense_prefill = (
|
||||
metadata.prefill is not None and metadata.prefill.use_dense_mha
|
||||
)
|
||||
num_reqs = metadata.num_decodes if use_dense_prefill else metadata.num_reqs
|
||||
num_tokens = int(common_attn_metadata.query_start_loc_cpu[num_reqs].item())
|
||||
num_decodes = metadata.num_decodes
|
||||
num_prefills = metadata.num_prefills
|
||||
num_decode_tokens = metadata.num_decode_tokens
|
||||
starts = np.asarray(common_attn_metadata.query_start_loc_cpu, dtype=np.int32)
|
||||
seg_lengths = np.diff(starts)
|
||||
seg_lengths = np.diff(starts[: num_reqs + 1])
|
||||
req_id_per_token = np.repeat(
|
||||
np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths
|
||||
)
|
||||
@@ -616,15 +610,13 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
self.req_id_per_token_buffer[:new_req_extent].copy_(
|
||||
np_to_pinned_tensor(req_id_per_token), non_blocking=True
|
||||
)
|
||||
query_lens = (
|
||||
common_attn_metadata.query_start_loc[1:]
|
||||
- common_attn_metadata.query_start_loc[:-1]
|
||||
)
|
||||
seq_lens = common_attn_metadata.seq_lens
|
||||
query_start_loc = common_attn_metadata.query_start_loc[: num_reqs + 1]
|
||||
query_lens = query_start_loc[1:] - query_start_loc[:-1]
|
||||
seq_lens = common_attn_metadata.seq_lens[:num_reqs]
|
||||
sparse_seqlen = generate_sparse_seqlen_triton(
|
||||
query_lens,
|
||||
seq_lens,
|
||||
common_attn_metadata.query_start_loc,
|
||||
query_start_loc,
|
||||
self.topk_tokens,
|
||||
num_tokens,
|
||||
common_attn_metadata.max_query_len,
|
||||
@@ -663,8 +655,13 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
reduce_indptr = None
|
||||
reduce_final_map = None
|
||||
reduce_partial_map = None
|
||||
if self._use_persistent_metadata and not use_triton_sparse:
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
is_long_pure_bf16_prefill = _is_long_bf16_prefill(self._mla_kv_dtype, metadata)
|
||||
if (
|
||||
self._use_persistent_metadata
|
||||
and not use_triton_sparse
|
||||
and not is_long_pure_bf16_prefill
|
||||
and num_tokens > 0
|
||||
):
|
||||
with gpu_sync_allowed():
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens[:num_reqs].cpu().numpy()
|
||||
clamped_seq_lens = np.minimum(
|
||||
@@ -708,6 +705,8 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
uni_seqlen_qo=1,
|
||||
fast_mode=True,
|
||||
max_split_per_batch=max_split_per_batch,
|
||||
dtype_q=self._mla_q_dtype,
|
||||
dtype_kv=self._mla_kv_dtype,
|
||||
)
|
||||
torch.cuda.current_stream(self.device).synchronize()
|
||||
self._prev_metadata_key = metadata_key
|
||||
@@ -718,32 +717,18 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
reduce_final_map = self._mla_reduce_final_map
|
||||
reduce_partial_map = self._mla_reduce_partial_map
|
||||
|
||||
metadata = ROCMAiterMLASparseMetadata(
|
||||
num_reqs=common_attn_metadata.num_reqs,
|
||||
max_query_len=common_attn_metadata.max_query_len,
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
num_actual_tokens=common_attn_metadata.num_actual_tokens,
|
||||
query_start_loc=common_attn_metadata.query_start_loc,
|
||||
slot_mapping=common_attn_metadata.slot_mapping,
|
||||
block_table=common_attn_metadata.block_table_tensor,
|
||||
req_id_per_token=req_id_per_token,
|
||||
block_size=self.kv_cache_spec.block_size,
|
||||
attn_out_dtype=self.model_dtype,
|
||||
topk_tokens=self.topk_tokens,
|
||||
num_decodes=num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
qo_indptr=qo_indptr,
|
||||
paged_kv_last_page_len=paged_kv_last_page_len,
|
||||
paged_kv_indices=paged_kv_indices,
|
||||
paged_kv_indptr=paged_kv_indptr,
|
||||
work_meta_data=work_meta_data,
|
||||
work_indptr=work_indptr,
|
||||
work_info_set=work_info_set,
|
||||
reduce_indptr=reduce_indptr,
|
||||
reduce_final_map=reduce_final_map,
|
||||
reduce_partial_map=reduce_partial_map,
|
||||
)
|
||||
metadata.attn_out_dtype = self.model_dtype
|
||||
metadata.req_id_per_token = req_id_per_token
|
||||
metadata.qo_indptr = qo_indptr
|
||||
metadata.paged_kv_last_page_len = paged_kv_last_page_len
|
||||
metadata.paged_kv_indices = paged_kv_indices
|
||||
metadata.paged_kv_indptr = paged_kv_indptr
|
||||
metadata.work_meta_data = work_meta_data
|
||||
metadata.work_indptr = work_indptr
|
||||
metadata.work_info_set = work_info_set
|
||||
metadata.reduce_indptr = reduce_indptr
|
||||
metadata.reduce_final_map = reduce_final_map
|
||||
metadata.reduce_partial_map = reduce_partial_map
|
||||
return metadata
|
||||
|
||||
|
||||
@@ -776,11 +761,7 @@ def reference_mla_sparse_prefill(
|
||||
return (result, lse)
|
||||
|
||||
|
||||
class ROCMAiterMLASparseImpl(
|
||||
MLAAttentionImpl[ROCMAiterMLASparseMetadata], SharedTopkIndicesBuffer
|
||||
):
|
||||
is_sparse = True
|
||||
supports_dense_mha_prefill = False
|
||||
class ROCMAiterMLASparseImpl(SparseMLACommonImpl[ROCMAiterMLASparseMetadata]):
|
||||
supports_dcp = False
|
||||
use_aiter_sparse_mla = False
|
||||
|
||||
@@ -803,10 +784,6 @@ class ROCMAiterMLASparseImpl(
|
||||
) -> None:
|
||||
AiterMLAHelper.check_num_heads_validity(num_heads)
|
||||
|
||||
self.num_heads = num_heads
|
||||
self.head_size = head_size
|
||||
self.scale = float(scale)
|
||||
self.num_kv_heads = num_kv_heads
|
||||
sinks = mla_args.pop("sinks", None)
|
||||
if sinks is not None:
|
||||
if sinks.dtype != torch.float32:
|
||||
@@ -821,10 +798,24 @@ class ROCMAiterMLASparseImpl(
|
||||
if not sinks.is_contiguous():
|
||||
raise ValueError("ROCm AITER MLA sinks must be contiguous")
|
||||
self.sinks: torch.Tensor | None = sinks
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.kv_lora_rank: int = mla_args["kv_lora_rank"]
|
||||
super().__init__(
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
num_kv_heads,
|
||||
alibi_slopes,
|
||||
sliding_window,
|
||||
kv_cache_dtype,
|
||||
logits_soft_cap,
|
||||
attn_type,
|
||||
kv_sharing_target_layer_name,
|
||||
indexer=indexer,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
**mla_args,
|
||||
)
|
||||
self.softmax_scale = scale
|
||||
self.init_topk_indices_buffer(indexer, topk_indices_buffer)
|
||||
self.supports_dense_mha_prefill = sinks is None # type: ignore[misc]
|
||||
self.masked_mha_available = False
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
@@ -894,14 +885,26 @@ class ROCMAiterMLASparseImpl(
|
||||
)
|
||||
)
|
||||
|
||||
if triton_sink_fallback or _use_rocm_sparse_triton(
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
head_size=q.shape[-1],
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
num_prefills=attn_metadata.num_prefills,
|
||||
num_decodes=attn_metadata.num_decodes,
|
||||
num_decode_tokens=attn_metadata.num_decode_tokens,
|
||||
max_query_len=attn_metadata.max_query_len,
|
||||
# Long BF16 prefills omit persistent metadata. gfx942's
|
||||
# nonpersistent ASM path is unsupported or unsafe for these shapes.
|
||||
# Sink attention has its own Triton/LSE dispatch below.
|
||||
triton_prefill_fallback = (
|
||||
not need_lse
|
||||
and on_gfx942()
|
||||
and _is_long_bf16_prefill(kv_c_and_k_pe_cache.dtype, attn_metadata)
|
||||
)
|
||||
if (
|
||||
triton_sink_fallback
|
||||
or triton_prefill_fallback
|
||||
or _use_rocm_sparse_triton(
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
head_size=q.shape[-1],
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
num_prefills=attn_metadata.num_prefills,
|
||||
num_decodes=attn_metadata.num_decodes,
|
||||
num_decode_tokens=attn_metadata.num_decode_tokens,
|
||||
max_query_len=attn_metadata.max_query_len,
|
||||
)
|
||||
):
|
||||
output = torch.empty(
|
||||
[num_tokens, q.shape[1], self.kv_lora_rank],
|
||||
@@ -1078,6 +1081,7 @@ class ROCMAiterMLASparseImpl(
|
||||
# Empty ragged rows have only sink mass and no value contribution.
|
||||
# AITER can return NaN output/LSE for those rows; do not multiply it
|
||||
# by a zero normalization factor and propagate the NaN.
|
||||
assert attn_metadata.paged_kv_indptr is not None
|
||||
has_keys = (
|
||||
attn_metadata.paged_kv_indptr[1:] > attn_metadata.paged_kv_indptr[:-1]
|
||||
).unsqueeze(-1)
|
||||
@@ -1156,7 +1160,7 @@ class ROCMAiterMLASparseImpl(
|
||||
else:
|
||||
ops.concat_mla_q(ql_nope, q_pe, q)
|
||||
|
||||
num_actual_toks = attn_metadata.num_actual_tokens
|
||||
num_actual_toks = q.shape[0]
|
||||
|
||||
# Get topk indices
|
||||
assert self.topk_indices_buffer is not None
|
||||
@@ -1168,7 +1172,7 @@ class ROCMAiterMLASparseImpl(
|
||||
kv_c_and_k_pe_cache, attn_metadata.block_size
|
||||
)
|
||||
triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token,
|
||||
attn_metadata.req_id_per_token[:num_actual_toks],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
attn_metadata.paged_kv_indptr,
|
||||
|
||||
Reference in New Issue
Block a user