From 50b21a2667b6903d2e0d651aab9f449832e5e1e9 Mon Sep 17 00:00:00 2001 From: TJian Date: Mon, 12 Oct 2026 23:50:28 +0800 Subject: [PATCH] [Perf] [Feat] [ROCm] Add densemha support to ROCm AITER Sparse MLA (#49263) Signed-off-by: tjtanaa --- .buildkite/test-amd.yaml | 10 +- .buildkite/test_areas/attention.yaml | 9 +- ...est_rocm_aiter_mla_sparse_metadata_sync.py | 503 ++++++++++++++++++ .../v1/attention/test_rocm_glm5next_sparse.py | 80 ++- .../v1/attention/test_sparse_mla_backends.py | 275 +++++++++- vllm/models/deepseek_v32/amd/rocm.py | 35 +- .../backends/mla/rocm_aiter_mla_sparse.py | 214 ++++---- 7 files changed, 984 insertions(+), 142 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index a14a0a33a4..b4c4b59fdf 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2632,8 +2632,11 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + - vllm/models/deepseek_v32/amd/rocm.py commands: - - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s v1/attention --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + + - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI300) Metrics, Tracing" # TBD timeout_in_minutes: 30 @@ -5642,10 +5645,11 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + - vllm/models/deepseek_v32/amd/rocm.py optional: true commands: - - pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI355 DPX) CUDAGraph" timeout_in_minutes: 55 diff --git a/.buildkite/test_areas/attention.yaml b/.buildkite/test_areas/attention.yaml index 83ba90e2cb..ec47e45fe7 100644 --- a/.buildkite/test_areas/attention.yaml +++ b/.buildkite/test_areas/attention.yaml @@ -30,6 +30,10 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + - vllm/models/deepseek_v32/amd/rocm.py + commands: + - pytest -v -s v1/attention --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":nvidia: (B200) V1 Attention Shard %N" key: v1-attention-b200 @@ -59,9 +63,10 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + - vllm/models/deepseek_v32/amd/rocm.py commands: - - pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --ignore=v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py v1/attention/test_sparse_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":nvidia: (DGX) Spark FlashInfer NVFP4 XQA" key: flashinfer-nvfp4-xqa-dgx-spark diff --git a/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py b/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py index aabce97fac..dc1ff79eea 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py @@ -33,6 +33,101 @@ class _FakeAiter(ModuleType): get_mla_metadata_v1: Mock +@pytest.mark.parametrize("block_size", [16, 64, 128]) +@pytest.mark.parametrize("topk_tokens", [128, 256]) +def test_rocm_index_conversion_matrix(block_size, topk_tokens): + """Convert valid, masked, and out-of-range sparse indices exactly.""" + device = torch.device("cuda") + num_tokens = 4 + max_blocks = 4 + req_ids = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device=device) + block_table = torch.tensor( + [[3, 5, 7, 9], [2, 4, 6, 8]], dtype=torch.int32, device=device + ) + token_indices = torch.arange( + num_tokens * topk_tokens, dtype=torch.int32, device=device + ).view(num_tokens, topk_tokens) + token_indices %= max_blocks * block_size + token_indices[0, 0] = -1 + token_indices[1, 1] = max_blocks * block_size + cu_seqlens = torch.arange( + 0, + (num_tokens + 1) * topk_tokens, + topk_tokens, + dtype=torch.int32, + device=device, + ) + output = torch.full( + (num_tokens * topk_tokens,), -99, dtype=torch.int32, device=device + ) + + sparse_mod.triton_convert_req_index_to_global_index( + req_ids, + block_table, + token_indices, + cu_seqlens, + output, + BLOCK_STRIDE_ROWS=block_size, + BLOCK_SIZE=block_size, + NUM_TOPK_TOKENS=topk_tokens, + ) + + block_ids = token_indices // block_size + valid = (token_indices >= 0) & (block_ids < max_blocks) + safe_blocks = block_ids.clamp(0, max_blocks - 1) + bases = block_table[req_ids[:, None].long(), safe_blocks.long()] + expected = torch.where(valid, bases * block_size + token_indices % block_size, 0) + torch.testing.assert_close(output.view_as(expected), expected, rtol=0, atol=0) + + +def test_rocm_index_conversion_rejects_mismatched_token_extents(): + """Reject full-batch request IDs paired with decode-subset indices.""" + with pytest.raises(AssertionError, match="must cover the same tokens"): + sparse_mod.triton_convert_req_index_to_global_index( + torch.zeros(3, dtype=torch.int32), + torch.zeros(1, 4, dtype=torch.int32), + torch.zeros(2, 128, dtype=torch.int32), + torch.arange(3, dtype=torch.int32), + torch.zeros(256, dtype=torch.int32), + BLOCK_STRIDE_ROWS=64, + NUM_TOPK_TOKENS=128, + ) + + +def test_rocm_index_conversion_respects_packed_valid_counts(): + """ROCm's indptr contract packs only each token's valid top-k prefix.""" + device = torch.device("cuda") + topk_tokens = 128 + valid_counts = torch.tensor([1, 17, 128], dtype=torch.int32, device=device) + cu_seqlens = torch.zeros(4, dtype=torch.int32, device=device) + torch.cumsum(valid_counts, dim=0, out=cu_seqlens[1:]) + req_ids = torch.tensor([0, 0, 0], dtype=torch.int32, device=device) + block_table = torch.tensor([[2, 3, 4]], dtype=torch.int32, device=device) + token_indices = torch.arange( + 3 * topk_tokens, dtype=torch.int32, device=device + ).view(3, topk_tokens) + token_indices %= 3 * 64 + output = torch.full((int(cu_seqlens[-1]),), -99, dtype=torch.int32, device=device) + + sparse_mod.triton_convert_req_index_to_global_index( + req_ids, + block_table, + token_indices, + cu_seqlens, + output, + BLOCK_STRIDE_ROWS=64, + BLOCK_SIZE=64, + NUM_TOPK_TOKENS=topk_tokens, + ) + + expected_parts = [] + for token_id, count in enumerate(valid_counts.tolist()): + local = token_indices[token_id, :count] + expected_parts.append(block_table[0, (local // 64).long()] * 64 + local % 64) + expected = torch.cat(expected_parts) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + + def _make_builder(): builder = object.__new__(sparse_mod.ROCMAiterMLASparseMetadataBuilder) max_num_batched_tokens = 8 @@ -44,6 +139,18 @@ def _make_builder(): builder.kv_cache_dtype = "fp8" builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64) builder.topk_tokens = topk_tokens + # SparseMLACommonMetadataBuilder state consumed by its build() method. + builder.cp_kv_cache_interleave_size = 1 + builder.use_pcp = False + builder._prefill_backend = None + + def cpu_token_to_req(metadata): + return torch.repeat_interleave( + torch.arange(metadata.num_reqs, dtype=torch.int32), + metadata.query_start_loc_cpu.diff(), + ) + + builder._build_req_id_per_token = cpu_token_to_req # type: ignore[method-assign] builder.req_id_per_token_buffer = torch.zeros( max_num_batched_tokens, dtype=torch.int32, device="cpu" ) @@ -62,6 +169,8 @@ def _make_builder(): builder._use_persistent_metadata = True builder._num_attention_heads = 16 builder._num_compute_units = current_platform.num_compute_units() + builder._mla_q_dtype = torch.bfloat16 + builder._mla_kv_dtype = torch.bfloat16 builder._mla_work_meta_data = torch.empty(1, dtype=torch.int32, device="cpu") builder._mla_work_indptr = torch.empty(1, dtype=torch.int32, device="cpu") builder._mla_work_info_set = torch.empty(1, dtype=torch.int32, device="cpu") @@ -216,6 +325,8 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch): assert events == ["metadata", "sync"] assert fake_get_mla_metadata_v1_mock.call_count == 1 assert fake_get_mla_metadata_v1_mock.call_args.kwargs["max_split_per_batch"] == 1 + assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_q"] == torch.bfloat16 + assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_kv"] == torch.bfloat16 events.clear() @@ -223,3 +334,395 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch): assert events == [] assert fake_get_mla_metadata_v1_mock.call_count == 1 + + +def test_dense_prefill_builds_sparse_metadata_for_decode_subset(monkeypatch): + """Dense prefill must exclude prefill tokens from AITER sparse metadata. + + A mixed batch is routed as sparse MQA for its leading decode requests and + dense MHA for its trailing short prefill requests. The AITER metadata + buffers therefore must describe only the decode subset; including prefill + tokens would mismatch the query passed to ``forward_mqa``. + """ + builder = _make_builder() + builder.vllm_config = SimpleNamespace( + attention_config=SimpleNamespace(sparse_mla_force_mqa=False) + ) + + query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32) + seq_lens = torch.tensor([16, 4], dtype=torch.int32) + common_metadata = SimpleNamespace( + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens, + max_query_len=4, + max_seq_len=16, + ) + metadata = sparse_mod.ROCMAiterMLASparseMetadata( + num_reqs=2, + max_query_len=4, + max_seq_len=16, + num_actual_tokens=5, + query_start_loc=query_start_loc, + slot_mapping=torch.arange(5), + block_table=torch.arange(16, dtype=torch.int32).view(2, 8), + req_id_per_token=torch.tensor([0, 1, 1, 1, 1], dtype=torch.int32), + topk_tokens=4, + num_decodes=1, + num_prefills=1, + num_decode_tokens=1, + seq_lens=seq_lens, + prefill_max_seq_len=4, + prefill=SimpleNamespace(use_dense_mha=True), + ) + + monkeypatch.setattr( + sparse_mod.SparseMLACommonMetadataBuilder, + "build", + lambda *args, **kwargs: metadata, + ) + sparse_call: dict[str, torch.Tensor] = {} + + def fake_generate_sparse_seqlen( + query_lens, sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q + ): + sparse_call.update( + query_lens=query_lens.clone(), + seq_lens=sparse_seq_lens.clone(), + query_start_loc=sparse_query_start_loc.clone(), + num_tokens=num_tokens, + ) + return torch.ones(num_tokens, dtype=torch.int32) + + monkeypatch.setattr( + sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen + ) + fake_aiter = _FakeAiter("aiter") + fake_aiter.get_mla_metadata_v1 = Mock() + monkeypatch.setitem(sys.modules, "aiter", fake_aiter) + monkeypatch.setattr( + sparse_mod.torch.cuda, + "current_stream", + lambda device=None: SimpleNamespace(synchronize=lambda: None), + ) + + result = builder.build(0, common_metadata) + + assert result.prefill is metadata.prefill + assert result.num_prefills == 1 + assert result.req_id_per_token.tolist() == [0] + assert result.qo_indptr.numel() == 2 + assert result.paged_kv_indptr.numel() == 2 + assert sparse_call["num_tokens"] == 1 + assert sparse_call["query_lens"].tolist() == [1] + assert sparse_call["seq_lens"].tolist() == [16] + assert sparse_call["query_start_loc"].tolist() == [0, 1] + + +def test_long_prefill_shares_full_sparse_metadata_with_decode(monkeypatch): + """A prefill longer than top-k must remain in shared sparse metadata. + + Dense MHA is only selected when the prefill sequence fits within the + sparse top-k window. Once it exceeds that window, both decode and prefill + are handled by sparse MQA, so every query token must be represented by the + same sparse paging metadata. + """ + builder = _make_builder() + builder.vllm_config = SimpleNamespace( + attention_config=SimpleNamespace(sparse_mla_force_mqa=False) + ) + + # One decode token followed by a five-token prefill. topk_tokens is four, + # so the prefill must not be split onto the dense-MHA path. + query_start_loc = torch.tensor([0, 1, 6], dtype=torch.int32) + seq_lens = torch.tensor([16, 5], dtype=torch.int32) + common_metadata = SimpleNamespace( + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens, + max_query_len=5, + max_seq_len=16, + ) + metadata = sparse_mod.ROCMAiterMLASparseMetadata( + num_reqs=2, + max_query_len=5, + max_seq_len=16, + num_actual_tokens=6, + query_start_loc=query_start_loc, + slot_mapping=torch.arange(6), + block_table=torch.arange(16, dtype=torch.int32).view(2, 8), + req_id_per_token=torch.tensor([0, 1, 1, 1, 1, 1], dtype=torch.int32), + topk_tokens=4, + num_decodes=1, + num_prefills=1, + num_decode_tokens=1, + seq_lens=seq_lens, + prefill_max_seq_len=5, + prefill=SimpleNamespace(use_dense_mha=False), + ) + + monkeypatch.setattr( + sparse_mod.SparseMLACommonMetadataBuilder, + "build", + lambda *args, **kwargs: metadata, + ) + sparse_call: dict[str, torch.Tensor] = {} + + def fake_generate_sparse_seqlen( + query_lens, sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q + ): + sparse_call.update( + query_lens=query_lens.clone(), + seq_lens=sparse_seq_lens.clone(), + query_start_loc=sparse_query_start_loc.clone(), + num_tokens=num_tokens, + ) + return torch.ones(num_tokens, dtype=torch.int32) + + monkeypatch.setattr( + sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen + ) + fake_aiter = _FakeAiter("aiter") + get_persistent_metadata = Mock() + fake_aiter.get_mla_metadata_v1 = get_persistent_metadata + monkeypatch.setitem(sys.modules, "aiter", fake_aiter) + monkeypatch.setattr( + sparse_mod.torch.cuda, + "current_stream", + lambda device=None: SimpleNamespace(synchronize=lambda: None), + ) + + result = builder.build(0, common_metadata) + + assert result.num_decodes == 1 + assert result.num_prefills == 1 + assert result.req_id_per_token.tolist() == [0, 1, 1, 1, 1, 1] + assert result.qo_indptr.numel() == 7 + assert result.paged_kv_indptr.numel() == 7 + assert result.paged_kv_indices.numel() == 6 * result.topk_tokens + assert sparse_call["num_tokens"] == 6 + assert sparse_call["query_lens"].tolist() == [1, 5] + assert sparse_call["seq_lens"].tolist() == [16, 5] + assert sparse_call["query_start_loc"].tolist() == [0, 1, 6] + get_persistent_metadata.assert_called_once() + assert result.work_meta_data is builder._mla_work_meta_data + assert result.reduce_indptr is builder._mla_reduce_indptr + + +def test_forward_mqa_slices_full_batch_metadata_to_query_tokens(monkeypatch): + """The sparse kernel inputs must have the same token extent as its query. + + In a mixed dense-prefill batch, ``forward_mqa`` receives only decode query + tokens while some metadata buffers may represent the full batch. This + guards the slicing that prevents the index conversion from writing beyond + the decode-sized output. + """ + impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl) + impl.kv_cache_dtype = "auto" + impl.num_heads = 4 + impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4) + expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16) + forward_mla = Mock(return_value=(expected_output, None)) + monkeypatch.setattr(impl, "_forward_mla", forward_mla) + + metadata = SimpleNamespace( + req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32), + block_table=torch.arange(40, dtype=torch.int32).view(4, 10), + paged_kv_indptr=torch.arange(3, dtype=torch.int32), + paged_kv_indices=torch.zeros(8, dtype=torch.int32), + block_size=1, + topk_tokens=4, + ) + query = torch.zeros(2, 4, 576, dtype=torch.bfloat16) + captured = {} + + def fake_convert(req_ids, block_table, token_indices, *args, **kwargs): + captured["req_ids"] = req_ids.clone() + captured["token_indices"] = token_indices.clone() + + monkeypatch.setattr( + sparse_mod, "triton_convert_req_index_to_global_index", fake_convert + ) + monkeypatch.setattr( + sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q + ) + + output, lse = impl.forward_mqa( + query, + torch.zeros(1, 1, 576, dtype=torch.bfloat16), + metadata, + SimpleNamespace(), + ) + + assert output is expected_output + assert lse is None + assert captured["req_ids"].tolist() == [0, 1] + assert captured["token_indices"].shape == (2, 4) + forward_mla.assert_called_once() + + +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +def test_forward_mqa_slices_decode_subset_for_bf16_and_fp8(monkeypatch, kv_cache_dtype): + """Both cache dtypes must slice mixed-batch metadata to decode queries.""" + impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl) + impl.kv_cache_dtype = kv_cache_dtype + impl.num_heads = 4 + impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4) + expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16) + forward_mla = Mock(return_value=(expected_output, None)) + monkeypatch.setattr(impl, "_forward_mla", forward_mla) + + metadata = SimpleNamespace( + req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32), + block_table=torch.arange(40, dtype=torch.int32).view(4, 10), + paged_kv_indptr=torch.arange(3, dtype=torch.int32), + paged_kv_indices=torch.zeros(8, dtype=torch.int32), + block_size=1, + topk_tokens=4, + ) + query = torch.zeros(2, 4, 576, dtype=torch.bfloat16) + kv_cache = torch.zeros(1, 1, 576, dtype=torch.uint8) + captured = {} + + def fake_convert(req_ids, block_table, token_indices, *args, **kwargs): + captured["req_ids"] = req_ids.clone() + captured["token_indices"] = token_indices.clone() + + monkeypatch.setattr( + sparse_mod, "triton_convert_req_index_to_global_index", fake_convert + ) + monkeypatch.setattr( + sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q + ) + if kv_cache_dtype == "fp8": + fp8_dtype = current_platform.fp8_dtype() + monkeypatch.setattr( + sparse_mod.ops, + "scaled_fp8_quant", + lambda q, scale: (q.to(fp8_dtype), None), + ) + + output, lse = impl.forward_mqa( + query, + kv_cache, + metadata, + SimpleNamespace(_q_scale=torch.tensor(1.0)), + ) + + assert output is expected_output + assert lse is None + assert captured["req_ids"].tolist() == [0, 1] + assert captured["token_indices"].shape == (2, 4) + + +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +@pytest.mark.parametrize("length_relation", ["short", "long"]) +@pytest.mark.parametrize("workload", ["prefill", "mixed", "decode"]) +def test_sparse_metadata_workload_length_dtype_matrix( + monkeypatch, workload, length_relation, kv_cache_dtype +): + """Cover routing and persistent metadata across the ROCm support matrix. + + The matrix crosses pure prefill, mixed prefill/decode, and decode-only + workloads with sequence lengths below/above top-k and BF16/FP8 KV cache. + """ + builder = _make_builder() + builder.vllm_config = SimpleNamespace( + attention_config=SimpleNamespace(sparse_mla_force_mqa=False) + ) + if kv_cache_dtype == "fp8": + builder._mla_q_dtype = current_platform.fp8_dtype() + builder._mla_kv_dtype = current_platform.fp8_dtype() + + is_long = length_relation == "long" + prefill_len = 5 if is_long else 3 + decode_seq_len = 8 if is_long else 3 + if workload == "prefill": + query_lens = [prefill_len] + seq_lens_list = [prefill_len] + num_decodes, num_prefills = 0, 1 + elif workload == "mixed": + query_lens = [1, prefill_len] + seq_lens_list = [decode_seq_len, prefill_len] + num_decodes, num_prefills = 1, 1 + else: + query_lens = [1, 1] + seq_lens_list = [decode_seq_len, decode_seq_len + 1] + num_decodes, num_prefills = 2, 0 + + query_start_loc = torch.tensor( + [0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32 + ) + seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32) + num_tokens = sum(query_lens) + common_metadata = SimpleNamespace( + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens, + max_query_len=max(query_lens), + max_seq_len=max(seq_lens_list), + ) + metadata = sparse_mod.ROCMAiterMLASparseMetadata( + num_reqs=len(query_lens), + max_query_len=max(query_lens), + max_seq_len=max(seq_lens_list), + num_actual_tokens=num_tokens, + query_start_loc=query_start_loc, + slot_mapping=torch.arange(num_tokens), + block_table=torch.arange(len(query_lens) * 8, dtype=torch.int32).view( + len(query_lens), 8 + ), + req_id_per_token=torch.repeat_interleave( + torch.arange(len(query_lens), dtype=torch.int32), + torch.tensor(query_lens), + ), + topk_tokens=4, + num_decodes=num_decodes, + num_prefills=num_prefills, + num_decode_tokens=num_decodes, + seq_lens=seq_lens, + prefill_max_seq_len=prefill_len if num_prefills else 0, + prefill=SimpleNamespace(use_dense_mha=not is_long) if num_prefills else None, + ) + monkeypatch.setattr( + sparse_mod.SparseMLACommonMetadataBuilder, + "build", + lambda *args, **kwargs: metadata, + ) + monkeypatch.setattr( + sparse_mod, + "generate_sparse_seqlen_triton", + lambda *args, **kwargs: torch.ones(args[4], dtype=torch.int32), + ) + get_persistent_metadata = Mock() + fake_aiter = _FakeAiter("aiter") + fake_aiter.get_mla_metadata_v1 = get_persistent_metadata + monkeypatch.setitem(sys.modules, "aiter", fake_aiter) + monkeypatch.setattr( + sparse_mod.torch.cuda, + "current_stream", + lambda device=None: SimpleNamespace(synchronize=lambda: None), + ) + + result = builder.build(0, common_metadata) + + uses_dense_prefill = workload in ("prefill", "mixed") and not is_long + expected_sparse_tokens = num_decodes if uses_dense_prefill else num_tokens + assert result.req_id_per_token.numel() == expected_sparse_tokens + assert result.qo_indptr.numel() == expected_sparse_tokens + 1 + assert result.paged_kv_indptr.numel() == expected_sparse_tokens + 1 + + uses_fallback = workload == "prefill" and is_long and kv_cache_dtype == "auto" + if expected_sparse_tokens == 0 or uses_fallback: + get_persistent_metadata.assert_not_called() + else: + get_persistent_metadata.assert_called_once() + if uses_fallback or expected_sparse_tokens == 0: + assert result.work_meta_data is None + assert result.reduce_indptr is None + else: + assert result.work_meta_data is builder._mla_work_meta_data + assert result.reduce_indptr is builder._mla_reduce_indptr diff --git a/tests/v1/attention/test_rocm_glm5next_sparse.py b/tests/v1/attention/test_rocm_glm5next_sparse.py index 6077b9eb2f..e4244bc06a 100644 --- a/tests/v1/attention/test_rocm_glm5next_sparse.py +++ b/tests/v1/attention/test_rocm_glm5next_sparse.py @@ -155,8 +155,86 @@ def test_rocm_sparse_triton_route( ) +@pytest.mark.parametrize( + ("gfx942", "fp8", "num_prefills", "num_decodes", "prefill_max_seq_len", "route"), + [ + pytest.param(True, False, 1, 0, 129, "triton", id="above-topk"), + pytest.param(False, False, 1, 0, 129, "aiter", id="gfx950"), + pytest.param(True, True, 1, 0, 129, "aiter", id="fp8"), + pytest.param(True, False, 0, 2, 129, "aiter", id="decode-only"), + pytest.param(True, False, 1, 1, 129, "aiter", id="mixed"), + pytest.param(True, False, 1, 0, 128, "aiter", id="equal-topk"), + pytest.param(True, False, 1, 0, 127, "aiter", id="below-topk"), + pytest.param(True, False, 0, 0, 129, "aiter", id="empty"), + pytest.param( + True, False, 1, 0, None, "missing", id="required-prefill-metadata" + ), + ], +) +def test_rocm_long_bf16_prefill_dispatch( + monkeypatch, gfx942, fp8, num_prefills, num_decodes, prefill_max_seq_len, route +): + """Check the new dispatch gate using CPU tensors and intercepted kernels.""" + monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942) + + class KernelSelected(Exception): + pass + + def triton_kernel(**kwargs): + raise KernelSelected("triton") + + def aiter_kernel(*args, **kwargs): + raise KernelSelected("aiter") + + monkeypatch.setattr(sparse_mod, "rocm_sparse_attn_prefill", triton_kernel) + monkeypatch.setattr(sparse_mod.rocm_aiter_ops, "mla_decode_fwd", aiter_kernel) + impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl) + impl.num_heads = 64 + impl.kv_lora_rank = 512 + impl.kv_cache_dtype = "fp8" if fp8 else "auto" + impl.scale = 576**-0.5 + impl.sinks = None + + dtype = torch.float8_e4m3fn if fp8 else torch.bfloat16 + q = torch.empty(2, 64, 576, dtype=dtype) + kv = torch.empty(4, 1, 576, dtype=dtype) + metadata = SimpleNamespace( + attn_out_dtype=torch.bfloat16, + num_prefills=num_prefills, + num_decodes=num_decodes, + num_decode_tokens=num_decodes, + max_query_len=2, + prefill_max_seq_len=prefill_max_seq_len, + topk_tokens=128, + work_meta_data=None, + qo_indptr=torch.arange(3, dtype=torch.int32), + paged_kv_indices=torch.empty(0, dtype=torch.int32), + paged_kv_indptr=torch.zeros(3, dtype=torch.int32), + paged_kv_last_page_len=torch.ones(2, dtype=torch.int32), + ) + # Irrelevant routes must not inspect prefill-only metadata. + if ( + not gfx942 + or fp8 + or num_decodes + or not num_prefills + or prefill_max_seq_len is None + ): + del metadata.prefill_max_seq_len + del metadata.topk_tokens + layer = SimpleNamespace(_q_scale=1.0, _k_scale=1.0) + error = AttributeError if route == "missing" else KernelSelected + match = "prefill_max_seq_len" if route == "missing" else f"^{route}$" + with pytest.raises(error, match=match): + impl._forward_mla(layer, q, kv, metadata) + + +@pytest.mark.parametrize("gfx942", [False, True]) @pytest.mark.parametrize("num_heads", [8, 12]) -def test_rocm_sparse_triton_route_preserves_padded_sinks(monkeypatch, num_heads): +def test_rocm_sparse_triton_route_preserves_padded_sinks( + monkeypatch, num_heads, gfx942 +): + monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942) captured = {} def fake_rocm_sparse_attn_prefill(**kwargs): diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 6eb089ce3e..c21fc15a52 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -47,13 +47,12 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import ( from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.platforms import current_platform -# TODO: Integrate ROCMAiterMLASparseBackend for ROCm. -# The ROCm sparse MLA backend (rocm_aiter_mla_sparse.py) has a compatible -# forward_mqa interface but needs validation on ROCm hardware. -if not current_platform.is_cuda(): +# This module's end-to-end sparse-backend correctness test supports CUDA and +# ROCm. The remaining tests exercise CUDA-specific FlashMLA/FlashInfer APIs +# and are skipped by the autouse fixture below when collecting on ROCm. +if not (current_platform.is_cuda() or current_platform.is_rocm()): pytest.skip( - "Sparse MLA backend tests currently only support CUDA. " - "ROCm support requires integrating ROCMAiterMLASparseBackend.", + "Sparse MLA backend tests require CUDA or ROCm.", allow_module_level=True, ) @@ -65,14 +64,16 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import ( ) from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import current_stream -from vllm.v1.attention.backends.mla import ( - flashattn_mla_sparse as flashattn_sparse_module, -) from vllm.v1.attention.backends.mla import index_group as index_group_module -from vllm.v1.attention.backends.mla.flashattn_mla_sparse import ( - FlashAttnMLASparseImpl, - FlashAttnMLASparseMetadataBuilder, -) + +if current_platform.is_cuda(): + from vllm.v1.attention.backends.mla import ( + flashattn_mla_sparse as flashattn_sparse_module, + ) + from vllm.v1.attention.backends.mla.flashattn_mla_sparse import ( + FlashAttnMLASparseImpl, + FlashAttnMLASparseMetadataBuilder, + ) from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( FlashInferMLASparseImpl, FlashInferMLASparseTRTLLMBackend, @@ -111,9 +112,43 @@ from vllm.v1.hisparse.runtime import ( ) from vllm.v1.hisparse.types import SparseKVRowMirror +if current_platform.is_rocm(): + from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import ( + ROCMAiterMLASparseBackend, + ) + + SPARSE_BACKENDS = [ROCMAiterMLASparseBackend] + SPARSE_BACKEND_IDS = ["ROCMAiterMLA"] + SPARSE_KV_CACHE_DTYPES = ["auto", "fp8"] + SPARSE_BLOCK_SIZES = [64] +else: + SPARSE_BACKENDS = [FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend] + SPARSE_BACKEND_IDS = ["FlashMLA", "FlashInferTRTLLM"] + SPARSE_KV_CACHE_DTYPES = ["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"] + SPARSE_BLOCK_SIZES = [32, 64] + + +@pytest.fixture(autouse=True) +def _skip_cuda_specific_sparse_mla_tests_on_rocm(request): + """Skip tests whose helpers exercise CUDA-only sparse MLA APIs.""" + rocm_portable_tests = { + "test_sparse_backend_decode_correctness", + "test_sparse_backend_prefill_correctness", + "test_rocm_fused_v32_dense_prefill_matches_mqa", + "test_rocm_long_bf16_prefill_avoids_nonpersistent_aiter", + } + if ( + current_platform.is_rocm() + and getattr(request.node, "originalname", request.node.name) + not in rocm_portable_tests + ): + pytest.skip("This sparse MLA test exercises a CUDA-specific API") + + SPARSE_BACKEND_BATCH_SPECS = { name: BATCH_SPECS[name] for name in [ + "small_decode", "mixed_small", "mixed_medium", "small_prefill", @@ -128,10 +163,61 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_prefill"] = BatchSpec( SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec( seq_lens=[256] * 2, query_lens=[256] * 2 ) +SPARSE_BACKEND_BATCH_SPECS["large_decode_over_topk"] = BatchSpec( + seq_lens=[256, 384], query_lens=[1, 1] +) DEVICE_TYPE = current_platform.device_type +@pytest.mark.parametrize( + ("batch_name", "tensor_parallel_size"), + [ + ("large_q_prefill", 2), + ("large_q_pure_prefill", 4), + ("large_q_pure_prefill", 1), + ], +) +def test_rocm_long_bf16_prefill_avoids_nonpersistent_aiter( + default_vllm_config, + dist_init, + workspace_init, + monkeypatch, + batch_name, + tensor_parallel_size, +): + """MI300 long BF16 prefills must remain correct without AITER ASM dispatch.""" + if not current_platform.is_rocm(): + pytest.skip("ROCm sparse MLA regression") + from vllm._aiter_ops import rocm_aiter_ops + + # Exercise the gfx942 routing decision on other ROCm devices too; + # the numerical kernel still compiles for the actual device. + monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: True) + + def unsupported_aiter(*args, **kwargs): + pytest.fail("Long gfx942 BF16 prefill reached nonpersistent AITER") + + monkeypatch.setattr(rocm_aiter_ops, "mla_decode_fwd", unsupported_aiter) + test_sparse_backend_decode_correctness( + default_vllm_config=default_vllm_config, + dist_init=dist_init, + backend_cls=ROCMAiterMLASparseBackend, + batch_name=batch_name, + kv_cache_dtype="auto", + tensor_parallel_size=tensor_parallel_size, + block_size=64, + workspace_init=workspace_init, + q_scale=1.0, + k_scale=1.0, + total_num_heads=128, + qk_nope_head_dim=128, + qk_rope_head_dim=64, + v_head_dim=128, + monkeypatch=monkeypatch, + ) + + def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch): """Weight absorption must not change the model's attention temperature.""" model_scale = 256**-0.5 @@ -403,16 +489,13 @@ def _quantize_dequantize_nvfp4_ds_mla( @pytest.mark.parametrize( "backend_cls", - [FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend], - ids=["FlashMLA", "FlashInferTRTLLM"], + SPARSE_BACKENDS, + ids=SPARSE_BACKEND_IDS, ) @pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys())) -@pytest.mark.parametrize( - "kv_cache_dtype", - ["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"], -) +@pytest.mark.parametrize("kv_cache_dtype", SPARSE_KV_CACHE_DTYPES) @pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4]) -@pytest.mark.parametrize("block_size", [32, 64]) +@pytest.mark.parametrize("block_size", SPARSE_BLOCK_SIZES) @pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)]) @pytest.mark.parametrize( ("total_num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"), @@ -525,6 +608,12 @@ def test_sparse_backend_decode_correctness( "attn_module_list_cfg": [{"topk_tokens": topk_tokens}], }, ) + # This shared correctness test constructs a sparse SDPA reference for every + # query token. ROCm normally routes short prefills to dense MHA, whose + # output intentionally differs from that sparse reference. Keep this test + # on the sparse-MQA path; dense-prefill routing has dedicated ROCm tests. + if current_platform.is_rocm(): + vllm_config.attention_config.sparse_mla_force_mqa = True model_config = vllm_config.model_config model_config.hf_text_config = SimpleNamespace( index_topk=topk_tokens, @@ -737,7 +826,7 @@ def test_sparse_backend_decode_correctness( vllm_config=vllm_config, ) vllm_config.compilation_config.static_forward_context["placeholder"] = ( - SimpleNamespace(prefill_backend=prefill_backend) + SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace()) ) builder_cls = backend_cls.get_builder_cls() @@ -1384,11 +1473,13 @@ PREFILL_BATCH_SPECS = { @pytest.mark.skipif( - torch.cuda.get_device_capability()[0] < 10, + current_platform.is_cuda() and torch.cuda.get_device_capability()[0] < 10, reason="Sparse MLA forward_mha requires FA4 (SM100+)", ) @pytest.mark.parametrize("batch_name", list(PREFILL_BATCH_SPECS.keys())) -@pytest.mark.parametrize("kv_cache_dtype", ["auto"]) +@pytest.mark.parametrize( + "kv_cache_dtype", (["auto", "fp8"] if current_platform.is_rocm() else ["auto"]) +) @pytest.mark.parametrize( ("num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"), [ @@ -1408,11 +1499,19 @@ def test_sparse_backend_prefill_correctness( v_head_dim, workspace_init, ): - """Test dense and masked MHA across supported sparse MLA dimensions.""" - backend_cls = FlashMLASparseBackend + """Test the normal dense forward_mha path for sparse MLA short prefill.""" + backend_cls = ( + ROCMAiterMLASparseBackend + if current_platform.is_rocm() + else FlashMLASparseBackend + ) + if current_platform.is_rocm() and ( + batch_name.startswith("masked_mha") or qk_nope_head_dim != 128 + ): + pytest.skip("ROCm dense prefill requires the DeepSeek MLA layout") batch_spec = PREFILL_BATCH_SPECS[batch_name] - device = torch.device("cuda") + device = torch.device(DEVICE_TYPE) dtype = torch.bfloat16 block_size = 64 @@ -1579,7 +1678,7 @@ def test_sparse_backend_prefill_correctness( vllm_config=vllm_config, ) vllm_config.compilation_config.static_forward_context["placeholder"] = ( - SimpleNamespace(prefill_backend=prefill_backend) + SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace()) ) builder_cls = backend_cls.get_builder_cls() @@ -1653,7 +1752,10 @@ def test_sparse_backend_prefill_correctness( assert out_buffer.shape == ref_output.shape assert torch.isfinite(out_buffer).all(), "Non-finite values in output" - torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01) + if kv_cache_dtype.startswith("fp8"): + torch.testing.assert_close(out_buffer, ref_output, rtol=0.065, atol=0.05) + else: + torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01) @pytest.mark.parametrize( @@ -3057,7 +3159,7 @@ def test_hisparse_mixed_batch_bf16_row_split( vllm_config=vllm_config, ) vllm_config.compilation_config.static_forward_context["placeholder"] = ( - SimpleNamespace(prefill_backend=prefill_backend) + SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace()) ) builder_cls = FlashMLASparseBackend.get_builder_cls() @@ -4264,3 +4366,118 @@ def test_sparse_mla_common_impl_resolves_buffer_lazily(): assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), ( "topk_indices_buffer must stay a lazily-resolved property" ) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm model implementation") +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +def test_rocm_fused_v32_dense_prefill_matches_mqa( + tmp_path, vllm_runner, kv_cache_dtype +): + """Fused normalization, RoPE and mixed dispatch preserve generation/logprobs. + + Chunking unequal prompts exercises prefill alongside decode. The two long + prompts straddle the dense-prefill threshold. No pretrained weights or + tokenizer download is needed. + """ + import json + + from vllm import SamplingParams + + config = { + "architectures": ["DeepseekV32ForCausalLM"], + "attention_bias": False, + "attention_dropout": 0.0, + "bos_token_id": 0, + "eos_token_id": 1, + "ep_size": 1, + "first_k_dense_replace": 1, + "hidden_act": "silu", + "hidden_size": 2048, + "index_head_dim": 128, + "index_n_heads": 64, + "index_topk": 2048, + "initializer_range": 0.02, + "intermediate_size": 4096, + "kv_lora_rank": 512, + "max_position_embeddings": 8192, + "model_type": "deepseek_v32", + "moe_intermediate_size": 512, + "moe_layer_freq": 1, + "n_group": 1, + "n_routed_experts": 8, + "n_shared_experts": 1, + "norm_topk_prob": True, + "num_attention_heads": 32, + "num_experts_per_tok": 2, + "num_hidden_layers": 2, + "num_key_value_heads": 32, + "num_nextn_predict_layers": 0, + "q_lora_rank": 512, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "beta_fast": 32, + "beta_slow": 1, + "factor": 40, + "mscale": 1.0, + "mscale_all_dim": 1.0, + "original_max_position_embeddings": 4096, + "type": "yarn", + }, + "rope_theta": 10000, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "tie_word_embeddings": False, + "topk_group": 1, + "topk_method": "noaux_tc", + "torch_dtype": "bfloat16", + "transformers_version": "4.44.2", + "use_cache": True, + "v_head_dim": 128, + "vocab_size": 4096, + } + (tmp_path / "config.json").write_text(json.dumps(config)) + prompts = [ + {"prompt_token_ids": [2 + (i * 17 + j * 13) % 4000 for i in range(n)]} + for j, n in enumerate([64, 128, 2048, 2049]) + ] + sampling = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True, logprobs=5) + results = [] + for force_mqa in [True, False]: + with vllm_runner( + str(tmp_path), + load_format="dummy", + skip_tokenizer_init=True, + dtype="bfloat16", + kv_cache_dtype=kv_cache_dtype, + seed=42, + max_model_len=4096, + max_num_seqs=4, + max_num_batched_tokens=256, + block_size=64, + enable_chunked_prefill=True, + enable_prefix_caching=False, + enforce_eager=True, + gpu_memory_utilization=0.25, + attention_config={ + "backend": "ROCM_AITER_MLA_SPARSE", + "sparse_mla_force_mqa": force_mqa, + }, + ) as runner: + outputs = runner.llm.generate(prompts, sampling, use_tqdm=False) + results.append([request.outputs[0] for request in outputs]) + + for reference, actual in zip(*results): + assert len(actual.token_ids) == 16 + assert actual.token_ids == reference.token_ids + assert actual.logprobs is not None and reference.logprobs is not None + for token, expected, observed in zip( + actual.token_ids, reference.logprobs, actual.logprobs + ): + torch.testing.assert_close( + torch.tensor(observed[token].logprob), + torch.tensor(expected[token].logprob), + rtol=0.01, + atol=0.01, + ) diff --git a/vllm/models/deepseek_v32/amd/rocm.py b/vllm/models/deepseek_v32/amd/rocm.py index b9ac31f8be..7211edaa3a 100644 --- a/vllm/models/deepseek_v32/amd/rocm.py +++ b/vllm/models/deepseek_v32/amd/rocm.py @@ -1,11 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import cast + import torch from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer @@ -179,6 +182,8 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): else: attn_metadata = attn_metadata_raw + attn_metadata = cast(MLACommonMetadata | None, attn_metadata) + slot_mapping = forward_context.slot_mapping assert isinstance(slot_mapping, dict) mla_slot = slot_mapping.get(self.layer_name) @@ -213,6 +218,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): mla_kv_cache = self.kv_cache mla_k_scale = self._k_scale + use_mha = attn_metadata is not None and self._use_sparse_mha(attn_metadata) + kv_c_out = torch.empty_like(kv_c) if use_mha else None + k_pe_out = torch.empty_like(k_pe) if use_mha else None + q_c = fused_norm_rope( positions, q_c, @@ -237,9 +246,18 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): mla_k_scale=mla_k_scale, has_indexer=has_indexer, index_rope_interleave=self._index_rope_interleave, + kv_c_out=kv_c_out, + k_pe_out=k_pe_out, ) - ql_nope, q_pe = self._compute_ql_nope(q_c) + if use_mha: + q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim) + q_nope, q_pe = q.split( + [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1 + ) + ql_nope = q_nope + else: + ql_nope, q_pe = self._compute_ql_nope(q_c) if self.indexer is not None and not self.skip_topk: index_q = self.indexer.wq_b(q_c)[0] @@ -260,7 +278,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): indexer_n_head_scale, has_indexer=has_indexer, index_rope_interleave=self._index_rope_interleave, - quantize_mqa=self._fp8_kv, + quantize_mqa=self._fp8_kv and not use_mha, ) if self.indexer is not None and not self.skip_topk: @@ -270,6 +288,19 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): output.zero_() return + if use_mha: + assert kv_c_out is not None and k_pe_out is not None + mha_q = torch.cat((q_nope, mqa_q), dim=-1) + self.forward_impl( + mha_q, + kv_c_out, + k_pe_out.unsqueeze(1), + self.kv_cache, + attn_metadata, + output, + ) + return + num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined] kv_cache = self.kv_cache if self._fp8_kv_needs_view: diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index e30fec715d..a0393e772c 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -12,11 +12,10 @@ from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig, get_current_vllm_config from vllm.config.cache import CacheDType from vllm.logger import init_logger -from vllm.model_executor.layers.attention.mla_attention import ( - get_mla_dims, -) from vllm.model_executor.layers.attention.sparse_mla_attention import ( - SharedTopkIndicesBuffer, + SparseMLACommonImpl, + SparseMLACommonMetadata, + SparseMLACommonMetadataBuilder, ) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -26,10 +25,7 @@ from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, AttentionLayer, - AttentionMetadata, - AttentionMetadataBuilder, CommonAttentionMetadata, - MLAAttentionImpl, MultipleOf, ) from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( @@ -39,7 +35,6 @@ from vllm.v1.attention.backends.mla.sparse_utils import ( align_blocks_to_rows, flat_kv_row_view, ) -from vllm.v1.attention.backends.utils import split_decodes_and_prefills from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( rocm_sparse_attn_decode_bf16, rocm_sparse_attn_prefill, @@ -76,6 +71,18 @@ def _use_rocm_sparse_triton( ) +def _is_long_bf16_prefill( + kv_dtype: torch.dtype, + metadata: "ROCMAiterMLASparseMetadata", +) -> bool: + return ( + kv_dtype == torch.bfloat16 + and metadata.num_decodes == 0 + and metadata.num_prefills > 0 + and metadata.prefill_max_seq_len > metadata.topk_tokens + ) + + @triton.jit def _fit_kpool_indices_kernel( token_indices_ptr, # int32 [num_tokens, NUM_TOPK_TOKENS + TAIL_WIDTH] @@ -239,6 +246,10 @@ def triton_convert_req_index_to_global_index( assert req_id.dtype == torch.int32 assert block_table.dtype == torch.int32 assert token_indices.dtype == torch.int32 + assert req_id.shape[0] == token_indices.shape[0], ( + f"req_id ({req_id.shape[0]}) and token_indices ({token_indices.shape[0]}) " + "must cover the same tokens" + ) assert token_indices.shape[1] == NUM_TOPK_TOKENS assert NUM_TOPK_TOKENS % BLOCK_N == 0, ( f"NUM_TOPK_TOKENS ({NUM_TOPK_TOKENS}) must be divisible byBLOCK_N ({BLOCK_N})" @@ -390,35 +401,13 @@ class ROCMAiterMLASparseBackend(AttentionBackend): return on_mi3xx() -@dataclass -class ROCMAiterMLASparseMetadata(AttentionMetadata): - num_reqs: int - max_query_len: int - max_seq_len: int - - num_actual_tokens: int # Number of tokens excluding padding. - query_start_loc: torch.Tensor - slot_mapping: torch.Tensor - - block_table: torch.Tensor - req_id_per_token: torch.Tensor - - qo_indptr: torch.Tensor - paged_kv_last_page_len: torch.Tensor - paged_kv_indices: torch.Tensor - paged_kv_indptr: torch.Tensor - attn_out_dtype: torch.dtype - - block_size: int = 1 - topk_tokens: int = 2048 - - # Fields read by the shared MLA forward. This impl has no dense-MHA prefill - # path (supports_dense_mha_prefill=False), so it always runs the MQA path; - num_decodes: int = 0 - num_prefills: int = 0 - num_decode_tokens: int = 0 - prefill_max_seq_len: int = 0 - prefill: object = None +@dataclass(kw_only=True) +class ROCMAiterMLASparseMetadata(SparseMLACommonMetadata): + qo_indptr: torch.Tensor | None = None + paged_kv_last_page_len: torch.Tensor | None = None + paged_kv_indices: torch.Tensor | None = None + paged_kv_indptr: torch.Tensor | None = None + attn_out_dtype: torch.dtype | None = None # Persistent MLA metadata (only populated when persistent mode is enabled, # i.e. when the aiter sparse decode kernel supports work-stealing splits). @@ -432,9 +421,10 @@ class ROCMAiterMLASparseMetadata(AttentionMetadata): @dataclass class ROCMAiterMLASparseMetadataBuilder( - AttentionMetadataBuilder[ROCMAiterMLASparseMetadata] + SparseMLACommonMetadataBuilder[ROCMAiterMLASparseMetadata] ): _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + metadata_cls = ROCMAiterMLASparseMetadata def __init__( self, @@ -443,20 +433,15 @@ class ROCMAiterMLASparseMetadataBuilder( vllm_config: VllmConfig, device: torch.device, ): - self.kv_cache_spec = kv_cache_spec - self.model_config = vllm_config.model_config + super().__init__(kv_cache_spec, layer_names, vllm_config, device) self.model_dtype = vllm_config.model_config.dtype self.kv_cache_dtype = vllm_config.cache_config.cache_dtype parallel_config = vllm_config.parallel_config - self.device = device max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens - self.vllm_config = vllm_config self._init_reorder_batch_threshold(1, supports_spec_as_decode=True) self.num_heads = self.model_config.get_num_attention_heads(parallel_config) - self.mla_dims = get_mla_dims(self.model_config) - self.topk_tokens = vllm_config.model_config.hf_text_config.index_topk attention_context = vllm_config.compilation_config.static_forward_context # Sink decode must use AITER's nonpersistent path. In particular, # gfx942 has no persistent+LSE kernel, and its metadata heuristic @@ -521,9 +506,12 @@ class ROCMAiterMLASparseMetadataBuilder( kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto") if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"): kv_cache_dtype_str = "fp8" + q_dtype = dtypes.fp8 else: kv_cache_dtype_str = "bf16" kv_dtype = dtypes.d_dtypes.get(kv_cache_dtype_str, dtypes.bf16) + self._mla_q_dtype = q_dtype + self._mla_kv_dtype = kv_dtype ( (work_meta_data_size, work_meta_data_type), @@ -587,13 +575,19 @@ class ROCMAiterMLASparseMetadataBuilder( common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, ) -> ROCMAiterMLASparseMetadata: - num_tokens = common_attn_metadata.num_actual_tokens - (num_decodes, num_prefills, num_decode_tokens, _) = split_decodes_and_prefills( - common_attn_metadata, - decode_threshold=self.reorder_batch_threshold or 1, + metadata = super().build( + common_prefix_len, common_attn_metadata, fast_build=fast_build ) + use_dense_prefill = ( + metadata.prefill is not None and metadata.prefill.use_dense_mha + ) + num_reqs = metadata.num_decodes if use_dense_prefill else metadata.num_reqs + num_tokens = int(common_attn_metadata.query_start_loc_cpu[num_reqs].item()) + num_decodes = metadata.num_decodes + num_prefills = metadata.num_prefills + num_decode_tokens = metadata.num_decode_tokens starts = np.asarray(common_attn_metadata.query_start_loc_cpu, dtype=np.int32) - seg_lengths = np.diff(starts) + seg_lengths = np.diff(starts[: num_reqs + 1]) req_id_per_token = np.repeat( np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths ) @@ -616,15 +610,13 @@ class ROCMAiterMLASparseMetadataBuilder( self.req_id_per_token_buffer[:new_req_extent].copy_( np_to_pinned_tensor(req_id_per_token), non_blocking=True ) - query_lens = ( - common_attn_metadata.query_start_loc[1:] - - common_attn_metadata.query_start_loc[:-1] - ) - seq_lens = common_attn_metadata.seq_lens + query_start_loc = common_attn_metadata.query_start_loc[: num_reqs + 1] + query_lens = query_start_loc[1:] - query_start_loc[:-1] + seq_lens = common_attn_metadata.seq_lens[:num_reqs] sparse_seqlen = generate_sparse_seqlen_triton( query_lens, seq_lens, - common_attn_metadata.query_start_loc, + query_start_loc, self.topk_tokens, num_tokens, common_attn_metadata.max_query_len, @@ -663,8 +655,13 @@ class ROCMAiterMLASparseMetadataBuilder( reduce_indptr = None reduce_final_map = None reduce_partial_map = None - if self._use_persistent_metadata and not use_triton_sparse: - num_reqs = common_attn_metadata.num_reqs + is_long_pure_bf16_prefill = _is_long_bf16_prefill(self._mla_kv_dtype, metadata) + if ( + self._use_persistent_metadata + and not use_triton_sparse + and not is_long_pure_bf16_prefill + and num_tokens > 0 + ): with gpu_sync_allowed(): seq_lens_cpu = common_attn_metadata.seq_lens[:num_reqs].cpu().numpy() clamped_seq_lens = np.minimum( @@ -708,6 +705,8 @@ class ROCMAiterMLASparseMetadataBuilder( uni_seqlen_qo=1, fast_mode=True, max_split_per_batch=max_split_per_batch, + dtype_q=self._mla_q_dtype, + dtype_kv=self._mla_kv_dtype, ) torch.cuda.current_stream(self.device).synchronize() self._prev_metadata_key = metadata_key @@ -718,32 +717,18 @@ class ROCMAiterMLASparseMetadataBuilder( reduce_final_map = self._mla_reduce_final_map reduce_partial_map = self._mla_reduce_partial_map - metadata = ROCMAiterMLASparseMetadata( - num_reqs=common_attn_metadata.num_reqs, - max_query_len=common_attn_metadata.max_query_len, - max_seq_len=common_attn_metadata.max_seq_len, - num_actual_tokens=common_attn_metadata.num_actual_tokens, - query_start_loc=common_attn_metadata.query_start_loc, - slot_mapping=common_attn_metadata.slot_mapping, - block_table=common_attn_metadata.block_table_tensor, - req_id_per_token=req_id_per_token, - block_size=self.kv_cache_spec.block_size, - attn_out_dtype=self.model_dtype, - topk_tokens=self.topk_tokens, - num_decodes=num_decodes, - num_prefills=num_prefills, - num_decode_tokens=num_decode_tokens, - qo_indptr=qo_indptr, - paged_kv_last_page_len=paged_kv_last_page_len, - paged_kv_indices=paged_kv_indices, - paged_kv_indptr=paged_kv_indptr, - work_meta_data=work_meta_data, - work_indptr=work_indptr, - work_info_set=work_info_set, - reduce_indptr=reduce_indptr, - reduce_final_map=reduce_final_map, - reduce_partial_map=reduce_partial_map, - ) + metadata.attn_out_dtype = self.model_dtype + metadata.req_id_per_token = req_id_per_token + metadata.qo_indptr = qo_indptr + metadata.paged_kv_last_page_len = paged_kv_last_page_len + metadata.paged_kv_indices = paged_kv_indices + metadata.paged_kv_indptr = paged_kv_indptr + metadata.work_meta_data = work_meta_data + metadata.work_indptr = work_indptr + metadata.work_info_set = work_info_set + metadata.reduce_indptr = reduce_indptr + metadata.reduce_final_map = reduce_final_map + metadata.reduce_partial_map = reduce_partial_map return metadata @@ -776,11 +761,7 @@ def reference_mla_sparse_prefill( return (result, lse) -class ROCMAiterMLASparseImpl( - MLAAttentionImpl[ROCMAiterMLASparseMetadata], SharedTopkIndicesBuffer -): - is_sparse = True - supports_dense_mha_prefill = False +class ROCMAiterMLASparseImpl(SparseMLACommonImpl[ROCMAiterMLASparseMetadata]): supports_dcp = False use_aiter_sparse_mla = False @@ -803,10 +784,6 @@ class ROCMAiterMLASparseImpl( ) -> None: AiterMLAHelper.check_num_heads_validity(num_heads) - self.num_heads = num_heads - self.head_size = head_size - self.scale = float(scale) - self.num_kv_heads = num_kv_heads sinks = mla_args.pop("sinks", None) if sinks is not None: if sinks.dtype != torch.float32: @@ -821,10 +798,24 @@ class ROCMAiterMLASparseImpl( if not sinks.is_contiguous(): raise ValueError("ROCm AITER MLA sinks must be contiguous") self.sinks: torch.Tensor | None = sinks - self.kv_cache_dtype = kv_cache_dtype - self.kv_lora_rank: int = mla_args["kv_lora_rank"] + super().__init__( + num_heads, + head_size, + scale, + num_kv_heads, + alibi_slopes, + sliding_window, + kv_cache_dtype, + logits_soft_cap, + attn_type, + kv_sharing_target_layer_name, + indexer=indexer, + topk_indices_buffer=topk_indices_buffer, + **mla_args, + ) self.softmax_scale = scale - self.init_topk_indices_buffer(indexer, topk_indices_buffer) + self.supports_dense_mha_prefill = sinks is None # type: ignore[misc] + self.masked_mha_available = False vllm_config = get_current_vllm_config() max_tokens = vllm_config.scheduler_config.max_num_batched_tokens @@ -894,14 +885,26 @@ class ROCMAiterMLASparseImpl( ) ) - if triton_sink_fallback or _use_rocm_sparse_triton( - kv_cache_dtype=self.kv_cache_dtype, - head_size=q.shape[-1], - kv_lora_rank=self.kv_lora_rank, - num_prefills=attn_metadata.num_prefills, - num_decodes=attn_metadata.num_decodes, - num_decode_tokens=attn_metadata.num_decode_tokens, - max_query_len=attn_metadata.max_query_len, + # Long BF16 prefills omit persistent metadata. gfx942's + # nonpersistent ASM path is unsupported or unsafe for these shapes. + # Sink attention has its own Triton/LSE dispatch below. + triton_prefill_fallback = ( + not need_lse + and on_gfx942() + and _is_long_bf16_prefill(kv_c_and_k_pe_cache.dtype, attn_metadata) + ) + if ( + triton_sink_fallback + or triton_prefill_fallback + or _use_rocm_sparse_triton( + kv_cache_dtype=self.kv_cache_dtype, + head_size=q.shape[-1], + kv_lora_rank=self.kv_lora_rank, + num_prefills=attn_metadata.num_prefills, + num_decodes=attn_metadata.num_decodes, + num_decode_tokens=attn_metadata.num_decode_tokens, + max_query_len=attn_metadata.max_query_len, + ) ): output = torch.empty( [num_tokens, q.shape[1], self.kv_lora_rank], @@ -1078,6 +1081,7 @@ class ROCMAiterMLASparseImpl( # Empty ragged rows have only sink mass and no value contribution. # AITER can return NaN output/LSE for those rows; do not multiply it # by a zero normalization factor and propagate the NaN. + assert attn_metadata.paged_kv_indptr is not None has_keys = ( attn_metadata.paged_kv_indptr[1:] > attn_metadata.paged_kv_indptr[:-1] ).unsqueeze(-1) @@ -1156,7 +1160,7 @@ class ROCMAiterMLASparseImpl( else: ops.concat_mla_q(ql_nope, q_pe, q) - num_actual_toks = attn_metadata.num_actual_tokens + num_actual_toks = q.shape[0] # Get topk indices assert self.topk_indices_buffer is not None @@ -1168,7 +1172,7 @@ class ROCMAiterMLASparseImpl( kv_c_and_k_pe_cache, attn_metadata.block_size ) triton_convert_req_index_to_global_index( - attn_metadata.req_id_per_token, + attn_metadata.req_id_per_token[:num_actual_toks], attn_metadata.block_table, topk_indices, attn_metadata.paged_kv_indptr,