From c6566bb9f678679c6e42ad2beb45fece8b35356a Mon Sep 17 00:00:00 2001 From: Hexiang Wang <56632993+whx-sjtu@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:20:38 +0800 Subject: [PATCH] [ROCm][MLA] Keep DCP prefill context in FP8 through AllGather (#59070) Signed-off-by: whx-sjtu Co-authored-by: OpenAI Codex Co-authored-by: Claude Code Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> --- .buildkite/test-amd.yaml | 3 + .buildkite/test_areas/distributed.yaml | 3 + .../test_rocm_aiter_mla_op_registration.py | 20 + .../attention/test_rocm_aiter_mla_dcp_cprr.py | 7 + .../test_rocm_aiter_mla_mtp_split.py | 77 ++- .../v1/attention/test_rocm_mla_dcp_context.py | 445 ++++++++++++++++++ vllm/_aiter_ops.py | 52 ++ .../attention/backends/mla/rocm_aiter_mla.py | 152 +++++- .../attention/ops/rocm_aiter_mla_prefill.py | 105 +++++ 9 files changed, 847 insertions(+), 17 deletions(-) create mode 100644 tests/v1/attention/test_rocm_mla_dcp_context.py create mode 100644 vllm/v1/attention/ops/rocm_aiter_mla_prefill.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index e4ae7f897e..d2702e7d4f 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3539,8 +3539,11 @@ steps: - examples/rl/rlhf_async_new_apis.py - tests/distributed/test_weight_transfer.py - tests/distributed/test_packed_tensor.py + - tests/v1/attention/test_rocm_mla_dcp_context.py + - vllm/v1/attention/ops/rocm_aiter_mla_prefill.py commands: - pytest -v -s tests/distributed/test_context_parallel.py + - pytest -v -s tests/v1/attention/test_rocm_mla_dcp_context.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index f541c42c93..b30cbac0ce 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -528,8 +528,11 @@ steps: - examples/rl/rlhf_async_new_apis.py - tests/distributed/test_weight_transfer.py - tests/distributed/test_packed_tensor.py + - tests/v1/attention/test_rocm_mla_dcp_context.py + - vllm/v1/attention/ops/rocm_aiter_mla_prefill.py commands: - pytest -v -s tests/distributed/test_context_parallel.py + - pytest -v -s tests/v1/attention/test_rocm_mla_dcp_context.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization diff --git a/tests/kernels/attention/test_rocm_aiter_mla_op_registration.py b/tests/kernels/attention/test_rocm_aiter_mla_op_registration.py index 0944be8937..cb40c61c3b 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_op_registration.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_op_registration.py @@ -33,6 +33,26 @@ def _require_aiter(): pytest.skip("aiter is required on supported ROCm hardware for this test") +@torch.inference_mode() +def test_gather_kv_b_proj_op_schema() -> None: + """Validate fake tensors and mutation of both expanded K/V outputs.""" + _require_aiter() + from vllm._aiter_ops import rocm_aiter_ops # noqa: F401 + + rows, num_heads = 32, 12 + cache = torch.randn(rows, 1, 576, device="cuda").to(current_platform.fp8_dtype()) + scale = torch.tensor([0.07], device="cuda") + cu_seq_lens = torch.tensor([0, 13, rows], dtype=torch.int32, device="cuda") + indices = torch.arange(rows, dtype=torch.int32, device="cuda").flip(0) + weight = torch.randn(num_heads * 256, 512, dtype=torch.bfloat16, device="cuda") + k = torch.empty(rows, num_heads, 192, dtype=torch.bfloat16, device="cuda") + v = torch.empty(rows, num_heads, 128, dtype=torch.bfloat16, device="cuda") + opcheck( + torch.ops.vllm.rocm_aiter_gather_kv_b_proj, + (cache, scale, cu_seq_lens, indices, cu_seq_lens, weight, k, v), + ) + + @torch.inference_mode() def test_mla_decode_fwd_op_schema() -> None: """Opcheck validates registration, schema, fake-tensor, and ``o`` aliasing. diff --git a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py index cd88ade8ee..56886872ec 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py +++ b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py @@ -142,6 +142,13 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True): def init_common_builder(self, *args, **kwargs): self.num_heads = gathered_heads // DCP + self.q_data_type = torch.bfloat16 + self.mla_dims = SimpleNamespace( + kv_lora_rank=512, + qk_nope_head_dim=128, + qk_rope_head_dim=64, + v_head_dim=128, + ) self.dcp_world_size = DCP self.reorder_batch_threshold = 5 diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index da141acf68..1583995ac9 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -740,26 +740,34 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch): torch.testing.assert_close(output, reference, rtol=3e-2, atol=3e-2) -@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128]) @pytest.mark.parametrize( - "spec_method, parallel_drafting", + "num_heads,spec_method,parallel_drafting,group_cache_dtype,needs_row_map", [ - ("deepseek_mtp", False), - # A drafter that is not one of the historically recognized MTP methods, - # and a parallel one, so the threshold is 1 + 2 * num_spec rather than - # 1 + num_spec. Sizing the metadata off a method name instead leaves - # these at qlen=1 while the router still admits the full range. - ("custom", True), - ("eagle", False), + *[ + (num_heads, method, parallel, None, True) + for num_heads in [8, 16, 24, 32, 64, 128] + for method, parallel in [ + ("deepseek_mtp", False), + # Parallel drafting needs qlen=1+2*num_spec, regardless of name. + ("custom", True), + ("eagle", False), + ] + ], + pytest.param(16, "deepseek_mtp", False, "auto", False, id="bf16-group"), + pytest.param(16, "deepseek_mtp", False, "fp8_e5m2", False, id="e5m2-group"), ], ) -def test_mtp_builder_init_sizes_native_fp8_metadata( - monkeypatch, num_heads, spec_method, parallel_drafting +def test_builder_init_sizes_decode_metadata_and_filters_dcp_maps( + monkeypatch, + num_heads, + spec_method, + parallel_drafting, + group_cache_dtype, + needs_row_map, ): - """Aiter init sizes the metadata for every query length decode can be handed. + """Size decode metadata for padded heads/qlen; skip unsupported DCP maps. - Sweeping num_heads asserts metadata is sized for the padded decode shape, - covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding. + The group's raw cache format must override the global FP8 configuration. """ dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16") info_calls = [] @@ -789,6 +797,13 @@ def test_mtp_builder_init_sizes_native_fp8_metadata( def init_common_builder(self, *args, **kwargs): self.num_heads = num_heads + self.q_data_type = config.model_config.dtype + self.mla_dims = SimpleNamespace( + kv_lora_rank=512, + qk_nope_head_dim=128, + qk_rope_head_dim=64, + v_head_dim=128, + ) self.dcp_world_size = 1 self.cp_kv_cache_interleave_size = 1 # Mirror what _init_reorder_batch_threshold would have left behind: the @@ -813,6 +828,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata( init_common_builder, ) monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False) + monkeypatch.setattr(rocm_aiter_mla, "is_aiter_found_and_supported", lambda: True) config = SimpleNamespace( speculative_config=SimpleNamespace( @@ -840,7 +856,9 @@ def test_mtp_builder_init_sizes_native_fp8_metadata( ), ) builder = AiterMLAMetadataBuilder( - kv_cache_spec=SimpleNamespace(block_size=1, dtype=torch.bfloat16), + kv_cache_spec=SimpleNamespace( + block_size=1, dtype=torch.bfloat16, cache_dtype_str=group_cache_dtype + ), layer_names=["layer.0"], vllm_config=config, device=torch.device("cpu"), @@ -860,6 +878,35 @@ def test_mtp_builder_init_sizes_native_fp8_metadata( assert builder._mla_q_dtype == dtypes.fp8 assert builder._mla_kv_dtype == dtypes.fp8 + builder.dcp_world_size = 2 + chunk = object() + rows = torch.tensor([0], dtype=torch.int64) + build_rows = mock.Mock(return_value=rows) + monkeypatch.setattr(rocm_aiter_mla, "context_row_indices", build_rows) + monkeypatch.setattr( + rocm_aiter_mla, + "_supports_fused_dcp_prefill_config", + mock.Mock(side_effect=AssertionError("Support must be cached at init")), + ) + monkeypatch.setattr( + rocm_aiter_mla.MLACommonMetadataBuilder, + "build", + lambda *args: SimpleNamespace( + prefill=SimpleNamespace( + chunked_context=SimpleNamespace(chunks=[chunk]), + block_table=torch.empty(0), + ), + decode=None, + dcp_context_row_indices=None, + ), + ) + for _ in range(2): + metadata = builder.build(0, SimpleNamespace(causal=True)) + assert metadata.dcp_context_row_indices == ([rows] if needs_row_map else None) + assert build_rows.call_args_list == ( + [mock.call(chunk, torch.device("cpu"))] * 2 if needs_row_map else [] + ) + def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch): get_mla_metadata_v1 = mock.MagicMock() diff --git a/tests/v1/attention/test_rocm_mla_dcp_context.py b/tests/v1/attention/test_rocm_mla_dcp_context.py new file mode 100644 index 0000000000..c29576f93b --- /dev/null +++ b/tests/v1/attention/test_rocm_mla_dcp_context.py @@ -0,0 +1,445 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from tests.utils import multi_gpu_only +from vllm.platforms import current_platform +from vllm.v1.attention.ops.rocm_aiter_mla_prefill import context_row_indices + + +@pytest.mark.parametrize( + "padded,lengths,starts,total,expected", + [ + ([3, 2], [[7, 6], [2, 1]], [5, 0], 6, [0, 1, 5, 3, 4, 8]), + ([3], [[2, 9]], [4], 3, [3, 4, 5]), + ([3], [[1, 1]], [0], 3, None), + ], + ids=["padding_and_chunk_starts", "empty_rank", "inconsistent_total"], +) +def test_dcp_context_indices(padded, lengths, starts, total, expected): + chunk = SimpleNamespace( + padded_local_seq_lens=padded, + local_context_lens_allranks=lengths, + local_starts=starts, + num_local_context_tokens=sum(padded), + num_context_tokens=total, + ) + if expected is None: + with pytest.raises(AssertionError): + context_row_indices(chunk, torch.device("cpu")) + else: + rows = context_row_indices(chunk, torch.device("cpu")) + assert rows.tolist() == expected + assert rows.dtype == torch.int32 + + +def test_compressed_gather_preserves_bytes_and_workspace_partition(monkeypatch): + from vllm.v1.attention.ops import rocm_aiter_mla_prefill as prefill + + workspace = torch.full((12, 4), -1, dtype=torch.bfloat16) + cache = torch.arange(8, dtype=torch.uint8).reshape(2, 4) + chunk = SimpleNamespace( + num_local_context_tokens=2, + local_context_lens_allranks=[[2, 2]], + padded_local_cu_seq_lens=torch.tensor([0, 2]), + num_requests=1, + starts=torch.tensor([0]), + ) + + def local_gather(src, dst, *args): + assert src.dtype == dst.dtype == torch.uint8 + dst.copy_(src) + + def allgather(dst, src): + assert dst.dtype == src.dtype == torch.uint8 + assert dst.shape == (4, 4) + assert src.shape == (2, 4) + dst[:2].copy_(src) + dst[2:].copy_(src + 16) + + monkeypatch.setattr(prefill.ops, "cp_gather_cache", local_gather) + gathered = prefill.gather_compressed_context( + cache, workspace, torch.empty(1, 1), chunk, allgather, torch.float8_e4m3fn + ) + torch.testing.assert_close( + gathered.view(torch.uint8), torch.cat((cache, cache + 16)) + ) + # Retain the original logical capacity even though the byte view is larger. + assert gathered.data_ptr() - workspace.data_ptr() == 4 * 4 + torch.testing.assert_close(workspace.view(torch.uint8).reshape(-1, 4)[:2], cache) + + +@pytest.mark.parametrize( + "unsupported", + [ + "packed_cache", + "e5m2", + "bf16_cache", + "quantized_weight", + "bias", + "no_indices", + "query_dtype", + "prefill_dtype", + "weight_dtype", + "weight_layout", + "cache_layout", + "dimensions", + "no_aiter", + ], +) +def test_dcp_prefill_unsupported_formats_use_original_path(monkeypatch, unsupported): + from vllm.platforms import current_platform + + if not current_platform.is_rocm(): + pytest.skip("ROCm MLA backend") + from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( + AiterMLAImpl, + MLACommonImpl, + ) + + if unsupported == "no_aiter": + monkeypatch.setattr( + "vllm.v1.attention.backends.mla.rocm_aiter_mla." + "is_aiter_found_and_supported", + lambda: False, + ) + + impl = object.__new__(AiterMLAImpl) + impl.kv_cache_dtype = { + "packed_cache": "fp8_ds_mla", + "e5m2": "fp8_e5m2", + "bf16_cache": "auto", + }.get(unsupported, "fp8") + impl.kv_lora_rank = 512 + if unsupported == "dimensions": + impl.kv_lora_rank = 256 + impl.qk_nope_head_dim = impl.v_head_dim = 128 + impl.qk_rope_head_dim = 64 + impl.num_heads = 1 + impl.kv_b_proj = SimpleNamespace( + weight=torch.empty(256, 512, dtype=torch.bfloat16), + weight_scale=torch.ones(1) if unsupported == "quantized_weight" else None, + bias=torch.ones(256) if unsupported == "bias" else None, + ) + if unsupported == "weight_dtype": + impl.kv_b_proj.weight = impl.kv_b_proj.weight.half() + elif unsupported == "weight_layout": + impl.kv_b_proj.weight = torch.empty(512, 256, dtype=torch.bfloat16).T + metadata = SimpleNamespace( + prefill=SimpleNamespace( + q_data_type=torch.float16 + if unsupported == "prefill_dtype" + else torch.bfloat16 + ), + dcp_context_row_indices=None if unsupported == "no_indices" else [], + ) + expected = (torch.ones(1), torch.zeros(1)) + + def fused_mla_kv_concat(kv_nope, k_pe, use_fp8_prefill): + raise AssertionError("The fallback should receive the callback") + + def original_path(*args): + assert args[-1] is fused_mla_kv_concat + return expected + + monkeypatch.setattr( + MLACommonImpl, + "_context_parallel_compute_prefill_context", + original_path, + ) + q = torch.empty(1, 1, 192, dtype=torch.bfloat16) + cache = torch.empty(1, 1536, 576, dtype=torch.uint8) + if unsupported == "query_dtype": + q = q.half() + elif unsupported == "cache_layout": + cache = torch.empty(1, 1536, 1152, dtype=torch.uint8)[..., ::2] + assert ( + impl._context_parallel_compute_prefill_context( + q, + cache, + metadata, + torch.ones(1), + 8, + fused_mla_kv_concat_fn=fused_mla_kv_concat, + ) + is expected + ) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm MLA backend") +@multi_gpu_only(num_gpus=2) +@pytest.mark.parametrize("interleave", [1, 64]) +@pytest.mark.parametrize( + "context_lens,workspace_size", + [([145, 37], 1024), ([545, 0, 3, 0], 256)], + ids=["single_chunk", "continuations_and_empty_context"], +) +def test_dcp_prefill_matches_full_attention( + interleave, context_lens, workspace_size, tmp_path +): + """Real DCP gather, fused expansion and suffix/chunk merges match dense SDPA.""" + import torch.multiprocessing as mp + + from vllm._aiter_ops import is_aiter_found_and_supported + + if not is_aiter_found_and_supported(): + pytest.skip("AITER on CDNA 3 or newer is required") + mp.spawn( + _dcp_prefill_worker, + args=( + f"file://{tmp_path / 'dist_init'}", + interleave, + context_lens, + workspace_size, + ), + nprocs=2, + ) + + +@torch.inference_mode() +def _dcp_prefill_worker(rank, init_method, interleave, context_lens, workspace_size): + import torch.nn.functional as F + + from tests.v1.attention.test_mla_backends import MockMLAAttentionLayer + from tests.v1.attention.utils import ( + BatchSpec, + create_common_attn_metadata, + create_vllm_config, + ) + from vllm.config import set_current_vllm_config + from vllm.distributed import ( + cleanup_dist_env_and_memory, + init_distributed_environment, + initialize_model_parallel, + ) + from vllm.model_executor.layers.linear import ColumnParallelLinear + from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import ( + AiterFlashAttnPrefillBackend, + ) + from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( + AiterMLAImpl, + AiterMLAMetadataBuilder, + ) + from vllm.v1.attention.ops.dcp import MLADCPManager + from vllm.v1.kv_cache_interface import MLAAttentionSpec + from vllm.v1.worker.workspace import init_workspace_manager + + torch.accelerator.set_device_index(rank) + torch.set_num_threads(1) + device = torch.device(f"cuda:{rank}") + init_workspace_manager(device) + world_size, num_heads, block_size = 2, 12, 64 + config = create_vllm_config( + model_name="deepseek-ai/DeepSeek-V2-Lite", + dtype=torch.bfloat16, + tensor_parallel_size=world_size, + max_model_len=2048, + block_size=block_size, + max_num_seqs=8, + max_num_batched_tokens=2048, + ) + config.cache_config.cache_dtype = "fp8" + config.parallel_config.decode_context_parallel_size = world_size + config.parallel_config.cp_kv_cache_interleave_size = interleave + config.attention_config.use_prefill_query_quantization = False + config.kernel_config.enable_jit_warmup = False + with set_current_vllm_config(config): + init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=rank, + distributed_init_method=init_method, + ) + initialize_model_parallel( + tensor_model_parallel_size=world_size, + decode_context_model_parallel_size=world_size, + ) + try: + projection = ColumnParallelLinear( + 512, + num_heads * world_size * 256, + bias=False, + params_dtype=torch.bfloat16, + ).to(device) + torch.manual_seed(101 + rank) + projection.weight.copy_(torch.randn_like(projection.weight) * 0.02) + dims = dict( + kv_lora_rank=512, + qk_nope_head_dim=128, + qk_rope_head_dim=64, + v_head_dim=128, + ) + impl = AiterMLAImpl( + num_heads=num_heads, + head_size=576, + scale=192**-0.5, + num_kv_heads=1, + alibi_slopes=None, + sliding_window=None, + kv_cache_dtype="fp8", + logits_soft_cap=None, + attn_type="decoder", + kv_sharing_target_layer_name=None, + q_lora_rank=None, + qk_head_dim=192, + kv_b_proj=projection, + **dims, + ) + layer = MockMLAAttentionLayer( + impl=impl, + num_heads=num_heads, + device=device, + kv_b_proj=projection, + q_scale=1.0, + k_scale=0.07, + **dims, + ) + layer.prefill_backend = AiterFlashAttnPrefillBackend( + num_heads=num_heads, + scale=impl.scale, + vllm_config=config, + **dims, + ) + layer.dcp_manager = MLADCPManager( + config, + device, + num_heads, + 576, + 512, + torch.bfloat16, + torch.bfloat16, + None, + True, + False, + ) + config.compilation_config.static_forward_context["mla"] = layer + spec = MLAAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=576, + dtype=current_platform.fp8_dtype(), + cache_dtype_str="fp8", + ) + builder = AiterMLAMetadataBuilder(spec, ["mla"], config, device) + builder.chunked_prefill_workspace_size = workspace_size + builder.chunked_prefill_workspace = torch.empty( + (workspace_size + workspace_size // world_size, 576), + dtype=torch.bfloat16, + device=device, + ) + layer.dcp_manager.init_kv_gather( + builder.chunked_prefill_workspace, workspace_size + ) + query_lens = [i + 2 for i in range(len(context_lens))] + batch = BatchSpec( + seq_lens=[c + q for c, q in zip(context_lens, query_lens)], + query_lens=query_lens, + ) + common = create_common_attn_metadata( + batch, block_size, device, arange_block_indices=True + ) + metadata = builder.build(0, common) + assert metadata.prefill is not None + chunked = metadata.prefill.chunked_context + assert chunked is not None and metadata.dcp_context_row_indices is not None + if 0 in context_lens: + assert chunked.empty_token_slices + assert any(c.is_continuation for c in chunked.chunks) + else: + assert len(chunked.chunks) == 1 + blocks_per_req = common.block_table_tensor.shape[1] + cache = torch.zeros( + (len(context_lens) * blocks_per_req, block_size, 576), + dtype=torch.bfloat16, + device=device, + ).to(current_platform.fp8_dtype()) + queries, suffixes, references = [], [], [] + # All ranks reconstruct the same global cache before sharding it. + torch.manual_seed(42) + for request, (context_len, query_len) in enumerate( + zip(context_lens, query_lens) + ): + latent = torch.randn( + context_len, 576, device=device, dtype=torch.bfloat16 + ) + quantized = (latent.float() / layer._k_scale).to(cache.dtype) + positions = torch.arange(context_len, device=device) + shard = quantized.view(torch.uint8)[ + (positions // interleave) % world_size == rank + ] + cache.view(len(context_lens), -1, 576).view(torch.uint8)[ + request, : len(shard) + ].copy_(shard) + suffix = torch.randn( + query_len, 576, device=device, dtype=torch.bfloat16 + ) + q = ( + torch.randn( + query_len, num_heads, 192, device=device, dtype=torch.bfloat16 + ) + * 0.5 + ) + full_latent = torch.cat( + ( + (quantized.float() * layer._k_scale).to(torch.bfloat16), + suffix, + ) + ) + kv = F.linear(full_latent[:, :512], projection.weight).view( + -1, num_heads, 256 + ) + k_nope, v = kv.split(128, dim=-1) + k = torch.cat( + (k_nope, full_latent[:, None, 512:].expand(-1, num_heads, -1)), + dim=-1, + ) + mask = torch.arange(context_len + query_len, device=device)[ + None, : + ] <= (context_len + torch.arange(query_len, device=device)[:, None]) + references.append( + F.scaled_dot_product_attention( + q.transpose(0, 1).float(), + k.transpose(0, 1).float(), + v.transpose(0, 1).float(), + attn_mask=mask, + scale=impl.scale, + ).transpose(0, 1) + ) + queries.append(q) + suffixes.append(suffix) + q, suffix = torch.cat(queries), torch.cat(suffixes) + output = torch.empty( + q.shape[0], num_heads * 128, device=device, dtype=torch.bfloat16 + ) + # Ensure this test cannot silently succeed through the fallback. + with pytest.MonkeyPatch.context() as patch: + + def unexpected_fallback(*args, **kwargs): + raise AssertionError("Expected the compressed DCP path") + + patch.setattr( + "vllm.model_executor.layers.attention.mla_attention." + "MLACommonImpl._context_parallel_compute_prefill_context", + unexpected_fallback, + ) + impl.forward_mha( + q, + suffix[:, :512], + suffix[:, None, 512:], + cache, + metadata, + layer._k_scale, + output, + ) + torch.testing.assert_close( + output.view(-1, num_heads, 128).float(), + torch.cat(references), + rtol=0.03, + atol=0.003, + ) + finally: + cleanup_dist_env_and_memory() diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 8af999906f..0817c651a0 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -643,6 +643,36 @@ def _check_aiter_mla_fp8_support() -> bool: return _AITER_MLA_SUPPORTS_FP8 +def _rocm_aiter_gather_kv_b_proj_impl( + kv_cache: torch.Tensor, + kv_scale: torch.Tensor, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + cu_seq_lens: torch.Tensor, + weight: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, +) -> None: + from aiter.ops.triton.gather_kv_b_proj import gather_kv_b_proj + + gather_kv_b_proj( + kv_cache, kv_scale, kv_indptr, kv_indices, cu_seq_lens, weight, None, k, v + ) + + +def _rocm_aiter_gather_kv_b_proj_fake( + kv_cache: torch.Tensor, + kv_scale: torch.Tensor, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + cu_seq_lens: torch.Tensor, + weight: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, +) -> None: + pass + + def _rocm_aiter_mla_decode_fwd_impl( q: torch.Tensor, kv_buffer: torch.Tensor, @@ -2851,6 +2881,13 @@ class rocm_aiter_ops: dispatch_key=current_platform.dispatch_key, ) + direct_register_custom_op( + op_name="rocm_aiter_gather_kv_b_proj", + op_func=_rocm_aiter_gather_kv_b_proj_impl, + mutates_args=["k", "v"], + fake_impl=_rocm_aiter_gather_kv_b_proj_fake, + dispatch_key=current_platform.dispatch_key, + ) direct_register_custom_op( op_name="rocm_aiter_mla_decode_fwd", op_func=_rocm_aiter_mla_decode_fwd_impl, @@ -3465,6 +3502,21 @@ class rocm_aiter_ops: extra_indices=extra_kv_indices, ) + @staticmethod + def gather_kv_b_proj( + kv_cache: torch.Tensor, + kv_scale: torch.Tensor, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + cu_seq_lens: torch.Tensor, + weight: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + ) -> None: + torch.ops.vllm.rocm_aiter_gather_kv_b_proj( + kv_cache, kv_scale, kv_indptr, kv_indices, cu_seq_lens, weight, k, v + ) + @staticmethod def mla_decode_fwd( q: torch.Tensor, diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index 1115da630b..d64b4797c0 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -2,15 +2,16 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +from collections.abc import Callable from dataclasses import dataclass from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, ClassVar, Final +from typing import TYPE_CHECKING, ClassVar, Final, cast import torch from vllm import envs -from vllm._aiter_ops import rocm_aiter_ops +from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.distributed import get_dcp_group @@ -22,6 +23,8 @@ from vllm.model_executor.layers.attention.mla_attention import ( MLACommonMetadata, MLACommonMetadataBuilder, QueryLenSupport, + accumulate_mla_context_chunk, + init_mla_context_partial, ) from vllm.triton_utils import tl, triton from vllm.utils.gpu_sync_debug import gpu_sync_allowed @@ -38,6 +41,11 @@ from vllm.v1.attention.backends.utils import ( from vllm.v1.attention.ops.rocm_aiter_mla_merge import ( merge_mla_segments_triton, ) +from vllm.v1.attention.ops.rocm_aiter_mla_prefill import ( + context_row_indices, + expand_context, + gather_compressed_context, +) from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache if TYPE_CHECKING: @@ -46,6 +54,26 @@ if TYPE_CHECKING: logger = init_logger(__name__) +def _supports_fused_dcp_prefill_config( + kv_cache_dtype: str | None, + query_dtype: torch.dtype, + kv_lora_rank: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, +) -> bool: + """Whether the configuration supports fused DCP context expansion.""" + return ( + kv_cache_dtype in ("fp8", "fp8_e4m3") + and query_dtype == torch.bfloat16 + and kv_lora_rank == 512 + and qk_nope_head_dim == 128 + and qk_rope_head_dim == 64 + and v_head_dim == 128 + and is_aiter_found_and_supported() + ) + + def _segmented_mla_page_size(block_size: int) -> int: """Largest supported power-of-two subpage dividing a physical KV block. @@ -507,6 +535,7 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata): @dataclass class AiterMLAMetadata(MLACommonMetadata[AiterMLADecodeMetadata]): + dcp_context_row_indices: list[torch.Tensor] | None = None work_meta_data: torch.Tensor | None = None work_indptr: torch.Tensor | None = None work_info_set: torch.Tensor | None = None @@ -664,6 +693,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): device ).multi_processor_count + # Use the group's original format: decode normalizes E5M2 to "fp8". + kv_cache_dtype = getattr(kv_cache_spec, "cache_dtype_str", None) + if kv_cache_dtype is None: + kv_cache_dtype = vllm_config.cache_config.cache_dtype + self._supports_fused_dcp_prefill = _supports_fused_dcp_prefill_config( + kv_cache_dtype, + self.q_data_type, + self.mla_dims.kv_lora_rank, + self.mla_dims.qk_nope_head_dim, + self.mla_dims.qk_rope_head_dim, + self.mla_dims.v_head_dim, + ) + self.compilation_config = vllm_config.compilation_config self.decode_attn_out_dtype = vllm_config.model_config.dtype @@ -1577,6 +1619,17 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): attn_metadata = super().build( common_prefix_len, common_attn_metadata, fast_build ) + prefill = attn_metadata.prefill + if ( + prefill is not None + and prefill.chunked_context is not None + and self.dcp_world_size > 1 + and self._supports_fused_dcp_prefill + ): + attn_metadata.dcp_context_row_indices = [ + context_row_indices(chunk, prefill.block_table.device) + for chunk in prefill.chunked_context.chunks + ] if ( attn_metadata.decode is not None and attn_metadata.decode.has_persistent_metadata @@ -1975,6 +2028,101 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): self._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd self._mla_reduce_v1 = mla_reduce_v1 + def _context_parallel_compute_prefill_context( + self, + q: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + attn_metadata: MLACommonMetadata, + k_scale: torch.Tensor, + dcp_world_size: int, + fused_mla_kv_concat_fn: Callable[ + [torch.Tensor, torch.Tensor, bool], tuple[torch.Tensor, torch.Tensor] + ] + | None = None, + ): + attn_metadata = cast(AiterMLAMetadata, attn_metadata) + assert attn_metadata.prefill is not None + prefill = attn_metadata.prefill + weight = getattr(self.kv_b_proj, "weight", None) + eligible = ( + _supports_fused_dcp_prefill_config( + self.kv_cache_dtype, + prefill.q_data_type, + self.kv_lora_rank, + self.qk_nope_head_dim, + self.qk_rope_head_dim, + self.v_head_dim, + ) + and q.dtype == torch.bfloat16 + and weight is not None + and weight.dtype == torch.bfloat16 + and weight.shape == (self.num_heads * 256, 512) + and weight.is_contiguous() + and getattr(self.kv_b_proj, "bias", None) is None + and getattr(self.kv_b_proj, "weight_scale", None) is None + and kv_c_and_k_pe_cache.is_contiguous() + and attn_metadata.dcp_context_row_indices is not None + ) + if not eligible: + return super()._context_parallel_compute_prefill_context( + q, + kv_c_and_k_pe_cache, + attn_metadata, + k_scale, + dcp_world_size, + fused_mla_kv_concat_fn, + ) + from vllm.platforms import current_platform + + assert weight is not None + assert attn_metadata.dcp_context_row_indices is not None + chunked = prefill.chunked_context + assert chunked is not None and chunked.dcp_manager is not None + assert prefill.prefill_backend is not None + assert len(chunked.chunks) == len(attn_metadata.dcp_context_row_indices) + output = output_lse = None + for chunk, row_indices in zip( + chunked.chunks, attn_metadata.dcp_context_row_indices + ): + gathered = gather_compressed_context( + kv_c_and_k_pe_cache, + chunked.workspace, + prefill.block_table[chunk.request_slice], + chunk, + chunked.dcp_manager.kv_gather, + current_platform.fp8_dtype(), + ) + k, v = expand_context( + gathered, + k_scale, + row_indices, + chunk.cu_seq_lens, + weight, + self.num_heads, + self.qk_nope_head_dim, + self.qk_rope_head_dim, + self.v_head_dim, + ) + attn_output, attn_lse = prefill.prefill_backend.run_prefill_context_chunk( + chunk=chunk, + q=q[chunk.token_slice], + k=k, + v=v, + ) + if output is None: + if len(chunked.chunks) == 1 and not chunked.empty_token_slices: + return attn_output, attn_lse + output, output_lse = init_mla_context_partial( + chunked, + attn_output, + attn_lse, + num_tokens=q.shape[0], + ) + accumulate_mla_context_chunk( + chunk, attn_output, attn_lse, output, output_lse + ) + return output, output_lse + def _flash_attn_varlen_diff_headdims( self, q, k, v, return_softmax_lse=False, softmax_scale=None, **kwargs ): diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_prefill.py b/vllm/v1/attention/ops/rocm_aiter_mla_prefill.py new file mode 100644 index 0000000000..c6d839f724 --- /dev/null +++ b/vllm/v1/attention/ops/rocm_aiter_mla_prefill.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Compressed DCP context preparation for ROCm MLA.""" + +from collections.abc import Callable + +import torch + +from vllm import _custom_ops as ops +from vllm._aiter_ops import rocm_aiter_ops +from vllm.model_executor.layers.attention.mla_attention import MLACommonPrefillMetadata + + +def context_row_indices( + chunk: MLACommonPrefillMetadata.ContextChunk, device: torch.device +) -> torch.Tensor: + """Map compact request-major rows to the padded rank-major AllGather.""" + assert chunk.padded_local_seq_lens is not None + assert chunk.local_context_lens_allranks is not None + assert chunk.local_starts is not None + segments = [] + offset = 0 + for padded, lengths, start in zip( + chunk.padded_local_seq_lens, + chunk.local_context_lens_allranks, + chunk.local_starts, + ): + for rank, length in enumerate(lengths): + count = min(max(0, length - start), padded) + if count: + first = rank * chunk.num_local_context_tokens + offset + segments.append(torch.arange(first, first + count, dtype=torch.int32)) + offset += padded + rows = torch.cat(segments) if segments else torch.empty(0, dtype=torch.int32) + assert rows.numel() == chunk.num_context_tokens + return rows.to(device=device, non_blocking=True) + + +def gather_compressed_context( + cache: torch.Tensor, + workspace: torch.Tensor, + block_table: torch.Tensor, + chunk: MLACommonPrefillMetadata.ContextChunk, + gather: Callable[[torch.Tensor, torch.Tensor], object], + cache_dtype: torch.dtype, +) -> torch.Tensor: + """Gather FP8 bytes, preserving the existing workspace row capacity.""" + assert chunk.local_context_lens_allranks is not None + assert workspace.is_contiguous() + rows, width = workspace.shape + fp8_workspace = workspace.view(cache_dtype).view(-1, width)[:rows] + local_rows = chunk.num_local_context_tokens + world_size = len(chunk.local_context_lens_allranks[0]) + offset = rows // (world_size + 1) + assert offset * (world_size + 1) == rows + assert local_rows <= offset + local = fp8_workspace[:local_rows] + gathered = fp8_workspace[offset : offset + world_size * local_rows] + assert chunk.padded_local_cu_seq_lens is not None + ops.cp_gather_cache( + cache.view(torch.uint8), + local.view(torch.uint8), + block_table, + chunk.padded_local_cu_seq_lens, + chunk.num_requests, + chunk.starts, + ) + gather(gathered.view(torch.uint8), local.view(torch.uint8)) + return gathered + + +def expand_context( + gathered: torch.Tensor, + scale: torch.Tensor, + row_indices: torch.Tensor, + cu_seq_lens: torch.Tensor, + weight: torch.Tensor, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fuse row reorganization, dequantization, projection and K/V packing.""" + k = torch.empty( + (row_indices.numel(), num_heads, qk_nope_head_dim + qk_rope_head_dim), + device=gathered.device, + dtype=weight.dtype, + ) + v = torch.empty( + (row_indices.numel(), num_heads, v_head_dim), + device=gathered.device, + dtype=weight.dtype, + ) + if row_indices.numel(): + rocm_aiter_ops.gather_kv_b_proj( + gathered.unsqueeze(1), + scale, + cu_seq_lens, + row_indices, + cu_seq_lens, + weight, + k, + v, + ) + return k, v