From 49f2104c53c910033aae05524af120e7c39c5c48 Mon Sep 17 00:00:00 2001 From: shivampr Date: Wed, 24 Jun 2026 12:28:17 -0700 Subject: [PATCH] [Feature] Support DCP with FP8 KV cache in MLA decode path (#44044) Signed-off-by: shivampr Signed-off-by: Shivam Co-authored-by: Matthew Bonanni Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/kernels/attention/test_cache.py | 72 ++++++ tests/v1/attention/test_mla_backends.py | 215 ++++++++++++++++++ .../layers/attention/mla_attention.py | 82 +++++-- vllm/v1/attention/backends/mla/flashmla.py | 5 +- 4 files changed, 359 insertions(+), 15 deletions(-) diff --git a/tests/kernels/attention/test_cache.py b/tests/kernels/attention/test_cache.py index 4cbeb7a0b9..7558da1c60 100644 --- a/tests/kernels/attention/test_cache.py +++ b/tests/kernels/attention/test_cache.py @@ -1015,6 +1015,78 @@ def test_gather_and_maybe_dequant_cache_mla( torch.testing.assert_close(dst, expected) +@pytest.mark.parametrize("kv_lora_rank", [512]) +@pytest.mark.parametrize("qk_rope_head_dim", [64]) +@pytest.mark.parametrize("block_size", [16]) +@pytest.mark.parametrize("num_blocks", [128]) +@pytest.mark.parametrize("dtype", [torch.float32]) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +@pytest.mark.parametrize("device", CUDA_DEVICES) +@torch.inference_mode() +def test_gather_and_maybe_dequant_cache_mla_with_seq_starts( + kv_lora_rank, + qk_rope_head_dim, + block_size, + num_blocks, + dtype, + kv_cache_dtype, + device, +): + entry_size = kv_lora_rank + qk_rope_head_dim + scale = torch.tensor(0.1, dtype=torch.float32, device=device) + src_cache = _create_mla_cache( + num_blocks, block_size, entry_size, dtype, kv_cache_dtype, device + ) + _fill_mla_cache(src_cache, kv_cache_dtype=kv_cache_dtype) + + seq_starts = torch.tensor([3, 17, 5], dtype=torch.int32, device=device) + seq_lens = torch.tensor([20, 10, 16], dtype=torch.int32, device=device) + batch_size = seq_lens.shape[0] + total_tokens = seq_lens.sum().item() + cu_seq_lens = torch.empty((batch_size + 1), dtype=torch.int32, device=device) + cu_seq_lens[0] = 0 + cu_seq_lens[1:] = seq_lens.cumsum(dim=0) + token_to_seq = torch.repeat_interleave( + torch.arange(batch_size, dtype=torch.int32, device=device), seq_lens + ) + + block_table = torch.empty( + (batch_size, num_blocks), dtype=torch.int32, device=device + ) + for b in range(batch_size): + block_table[b, :] = torch.randperm(num_blocks, device=device) + + if kv_cache_dtype == "fp8": + dequant_src_cache = torch.empty_like(src_cache, dtype=dtype) + ops.convert_fp8(dequant_src_cache, src_cache, scale.item()) + else: + dequant_src_cache = src_cache + + expected_rows = [] + for b in range(batch_size): + start = seq_starts[b].item() + length = seq_lens[b].item() + for offset in range(start, start + length): + block_id = block_table[b, offset // block_size] + slot = offset % block_size + expected_rows.append(dequant_src_cache[block_id, slot]) + expected = torch.stack(expected_rows) + + dst = torch.zeros((total_tokens, entry_size), dtype=dtype, device=device) + ops.gather_and_maybe_dequant_cache( + src_cache, + dst, + block_table, + cu_seq_lens, + token_to_seq, + total_tokens, + kv_cache_dtype, + scale, + seq_starts, + ) + torch.testing.assert_close(dst, expected) + + @pytest.mark.parametrize("kv_lora_rank", [512]) @pytest.mark.parametrize("qk_rope_head_dim", [64]) @pytest.mark.parametrize("block_size", [16]) diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index 1ef4f96617..315c77de39 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -19,6 +19,7 @@ from tests.v1.attention.utils import ( ) from vllm import _custom_ops as ops from vllm.config.vllm import set_current_vllm_config +from vllm.model_executor.layers.attention import mla_attention as mla_attention_module from vllm.model_executor.layers.attention.mla_attention import ( MLAAttention, QueryLenSupport, @@ -30,6 +31,7 @@ from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla +from vllm.v1.attention.backends.mla import flashmla as flashmla_module from vllm.v1.attention.backends.mla.prefill import ( MLAPrefillBackendEnum, get_mla_prefill_backend, @@ -552,6 +554,10 @@ class MockMLAAttentionLayer(MLAAttention): ) else: mqa_q = (mqa_ql_nope, mqa_q_pe) + if self.impl.dcp_world_size > 1: + if isinstance(mqa_q, tuple): + mqa_q = torch.cat(mqa_q, dim=-1) + mqa_q = mla_attention_module.get_dcp_group().all_gather(mqa_q, dim=1) attn_out, _ = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self) @@ -569,6 +575,215 @@ class MockMLAAttentionLayer(MLAAttention): return output +def test_mock_mla_dcp_fp8_decode_gathers_quantized_query( + monkeypatch, default_vllm_config +): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for FP8 decode query quantization path.") + + device = torch.device(f"{DEVICE_TYPE}:0") + num_tokens = 2 + num_heads = 2 + qk_nope_head_dim = 4 + qk_rope_head_dim = 2 + v_head_dim = 3 + kv_lora_rank = 5 + + class _DummyKVProj: + def __init__(self): + # Shape expected by MockMLAAttentionLayer.__init__ + self.weight = torch.randn( + num_heads * (qk_nope_head_dim + v_head_dim), + kv_lora_rank, + device=device, + dtype=torch.float32, + ) + + class _FakeImpl: + def __init__(self): + self.kv_cache_dtype = "fp8" + self.supports_quant_query_input = True + self.dcp_world_size = 2 + self.forward_q = None + + def forward_mha(self, *args, **kwargs): + return None + + def forward_mqa(self, q, kv_cache, attn_metadata, layer): + self.forward_q = q + assert isinstance(q, torch.Tensor) + bsz, _, _ = q.shape + return ( + torch.zeros( + bsz, + num_heads, + kv_lora_rank, + device=q.device, + dtype=torch.float32, + ), + None, + ) + + class _FakeDCPGroup: + def __init__(self): + self.calls = 0 + self.input_dtype = None + self.input_shape = None + + def all_gather(self, x, dim=1): + self.calls += 1 + self.input_dtype = x.dtype + self.input_shape = tuple(x.shape) + return torch.cat([x, x], dim=dim) + + fake_group = _FakeDCPGroup() + monkeypatch.setattr(mla_attention_module, "get_dcp_group", lambda: fake_group) + + impl = _FakeImpl() + with set_current_vllm_config(default_vllm_config): + layer = MockMLAAttentionLayer( + impl=impl, + num_heads=num_heads, + qk_nope_head_dim=qk_nope_head_dim, + qk_rope_head_dim=qk_rope_head_dim, + v_head_dim=v_head_dim, + kv_lora_rank=kv_lora_rank, + device=device, + kv_b_proj=_DummyKVProj(), + q_scale=1.0, + k_scale=1.0, + ) + + q = torch.randn( + num_tokens, + num_heads, + qk_nope_head_dim + qk_rope_head_dim, + device=device, + dtype=torch.float32, + ) + kv_c = torch.randn(num_tokens, kv_lora_rank, device=device, dtype=torch.float32) + k_pe = torch.randn( + num_tokens, 1, qk_rope_head_dim, device=device, dtype=torch.float32 + ) + kv_cache = torch.empty(0, device=device, dtype=torch.float32) + output = torch.empty( + num_tokens, num_heads * v_head_dim, device=device, dtype=torch.float32 + ) + + class _AttnMeta: + num_decode_tokens = num_tokens + num_decodes = 1 + num_prefills = 0 + slot_mapping = torch.empty(0, dtype=torch.long, device=device) + + layer.forward_impl(q, kv_c, k_pe, kv_cache, _AttnMeta(), output) + + assert fake_group.calls == 1 + assert fake_group.input_dtype == current_platform.fp8_dtype() + assert fake_group.input_shape == ( + num_tokens, + num_heads, + kv_lora_rank + qk_rope_head_dim, + ) + assert isinstance(impl.forward_q, torch.Tensor) + assert tuple(impl.forward_q.shape) == ( + num_tokens, + num_heads * impl.dcp_world_size, + kv_lora_rank + qk_rope_head_dim, + ) + + +@pytest.mark.parametrize("is_fp8_kvcache", [False, True], ids=["bf16", "fp8"]) +def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads( + monkeypatch, is_fp8_kvcache +): + class _FakeSchedulerMetadata: + tile_scheduler_metadata = None + num_splits = None + + base_call: tuple[torch.Tensor, int, int, bool] | None = None + fp8_call: tuple[torch.Tensor, int, int] | None = None + + def fake_get_mla_metadata( + seq_lens_device, + num_q_tokens_per_head_k, + num_heads_k, + is_fp8_kvcache=False, + ): + nonlocal base_call + base_call = ( + seq_lens_device, + num_q_tokens_per_head_k, + num_heads_k, + is_fp8_kvcache, + ) + return _FakeSchedulerMetadata(), None + + def fake_get_mla_metadata_dense_fp8( + seq_lens_device, num_q_tokens_per_head_k, num_heads_k + ): + nonlocal fp8_call + fp8_call = ( + seq_lens_device, + num_q_tokens_per_head_k, + num_heads_k, + ) + return ( + torch.empty((0, 8), dtype=torch.int32), + torch.empty((0,), dtype=torch.int32), + ) + + monkeypatch.setattr(flashmla_module, "get_mla_metadata", fake_get_mla_metadata) + monkeypatch.setattr( + flashmla_module, + "get_mla_metadata_dense_fp8", + fake_get_mla_metadata_dense_fp8, + ) + + builder = object.__new__(flashmla_module.FlashMLAMetadataBuilder) + builder.num_q_heads = 4 + builder.dcp_world_size = 2 + builder.is_fp8_kvcache = is_fp8_kvcache + builder.compilation_config = type( + "_CompilationConfig", + (), + { + "cudagraph_mode": type( + "_CudaGraphMode", + (), + {"has_full_cudagraphs": lambda self: False}, + )() + }, + )() + + seq_lens = torch.tensor([16, 24], dtype=torch.int32) + query_start_loc = torch.tensor([0, 1, 2], dtype=torch.int32) + + metadata = builder._build_decode( + block_table_tensor=torch.empty((2, 1), dtype=torch.int32), + seq_lens_device=seq_lens, + max_seq_len=24, + query_start_loc_cpu=query_start_loc, + query_start_loc_device=query_start_loc, + num_decode_tokens=2, + dcp_tot_seq_lens_device=None, + ) + + assert base_call is not None + assert base_call[0] is seq_lens + assert base_call[1:] == (8, 1, is_fp8_kvcache) + if is_fp8_kvcache: + assert metadata.scheduler_metadata.tile_scheduler_metadata is not None + assert metadata.scheduler_metadata.num_splits is not None + assert fp8_call is not None + assert fp8_call[0] is seq_lens + assert fp8_call[1:] == (8, 1) + else: + assert metadata.scheduler_metadata.tile_scheduler_metadata is None + assert metadata.scheduler_metadata.num_splits is None + assert fp8_call is None + + def run_attention_backend( backend: AttentionBackendEnum, kv_cache_spec: MLAAttentionSpec, diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 051468ed14..4dd666f0c6 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -800,9 +800,9 @@ class MLAAttention(nn.Module, AttentionLayerBase): else: mqa_q = (mqa_ql_nope, mqa_q_pe) if self.impl.dcp_world_size > 1: - assert not fp8_attention, "DCP not support fp8 kvcache now." - # concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P) - mqa_q = torch.cat(mqa_q, dim=-1) + if isinstance(mqa_q, tuple): + # concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P) + mqa_q = torch.cat(mqa_q, dim=-1) # mqa_q do allgather in head dim. mqa_q = get_dcp_group().all_gather(mqa_q, dim=1) @@ -1263,6 +1263,7 @@ class MLACommonPrefillMetadata: padded_local_chunk_seq_lens: list[list[int]] | None = None local_context_lens_allranks: list[list[int]] | None = None padded_local_cu_seq_lens: torch.Tensor | None = None + padded_local_token_to_seq: torch.Tensor | None = None cu_seq_lens_lst: list[list[int]] | None = None chunk_size: int | None = None prefill_tokens_with_context: int | None = None @@ -1787,6 +1788,21 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): out=padded_local_cu_chunk_seq_lens_cpu[:, 1:], dtype=torch.int32, ) + max_padded_local_tokens_over_chunk = ( + padded_local_cu_chunk_seq_lens_cpu[:, -1].max().item() + ) + padded_local_token_to_seq_tensor_cpu = torch.zeros( + [num_chunks, max_padded_local_tokens_over_chunk], + dtype=torch.int32, + ) + for i in range(num_chunks): + chunk_token_to_seq_tensor = torch.repeat_interleave( + range_idx, padded_local_chunk_seq_lens[i] + ) + chunk_len = chunk_token_to_seq_tensor.shape[0] + padded_local_token_to_seq_tensor_cpu[i, :chunk_len] = ( + chunk_token_to_seq_tensor + ) prefill_tokens_with_context = None if num_prefills_with_context_cpu > 0: @@ -1811,6 +1827,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): padded_local_cu_seq_lens=padded_local_cu_chunk_seq_lens_cpu.to( device, non_blocking=True ), + padded_local_token_to_seq=padded_local_token_to_seq_tensor_cpu.to( + device, non_blocking=True + ), cu_seq_lens_lst=cu_seq_lens_cpu.tolist(), chunk_size=padded_local_max_context_chunk_across_ranks, prefill_tokens_with_context=prefill_tokens_with_context, @@ -2187,7 +2206,6 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): k_scale: torch.Tensor, dcp_world_size: int, ): - assert k_scale is None, "DCP not support scaled kvcache now." assert attn_metadata.prefill is not None prefill_metadata = attn_metadata.prefill assert prefill_metadata.prefill_backend is not None @@ -2195,9 +2213,11 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): assert prefill_metadata.chunked_context.padded_local_chunk_seq_lens is not None assert prefill_metadata.chunked_context.local_context_lens_allranks is not None assert prefill_metadata.chunked_context.padded_local_cu_seq_lens is not None + assert prefill_metadata.chunked_context.padded_local_token_to_seq is not None assert prefill_metadata.chunked_context.cu_seq_lens_lst is not None assert prefill_metadata.chunked_context.chunk_size is not None + use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype() output = None merge_output = None iters = len(prefill_metadata.chunked_context.seq_tot) @@ -2205,16 +2225,37 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): for i in range(iters): toks = prefill_metadata.chunked_context.seq_tot[i] - ops.cp_gather_cache( - src_cache=kv_c_and_k_pe_cache, - dst=workspace, - block_table=prefill_metadata.block_table, - cu_seq_lens=prefill_metadata.chunked_context.padded_local_cu_seq_lens[ - i - ], - batch_size=attn_metadata.num_prefills, - seq_starts=prefill_metadata.chunked_context.starts[i], + if toks == 0: + continue + padded_local_cu_seq_lens = ( + prefill_metadata.chunked_context.padded_local_cu_seq_lens[i] ) + if is_quantized_kv_cache(self.kv_cache_dtype) and ( + self.kv_cache_dtype != "fp8_ds_mla" + ): + assert k_scale is not None + ops.gather_and_maybe_dequant_cache( + src_cache=kv_c_and_k_pe_cache, + dst=workspace, + block_table=prefill_metadata.block_table, + cu_seq_lens=padded_local_cu_seq_lens, + token_to_seq=prefill_metadata.chunked_context.padded_local_token_to_seq[ + i + ], + num_tokens=toks, + kv_cache_dtype=self.kv_cache_dtype, + scale=k_scale, + seq_starts=prefill_metadata.chunked_context.starts[i], + ) + else: + ops.cp_gather_cache( + src_cache=kv_c_and_k_pe_cache, + dst=workspace, + block_table=prefill_metadata.block_table, + cu_seq_lens=padded_local_cu_seq_lens, + batch_size=attn_metadata.num_prefills, + seq_starts=prefill_metadata.chunked_context.starts[i], + ) # workspace # |------- N tokens --------|--------- N*dcp_size tokens ----------| # |<- use for local_gather ->|<--------- use for allgather -------->| @@ -2252,9 +2293,22 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): toks=toks, ) + kv_b_proj_w_dtype = ( + self.kv_b_proj.weight.dtype + if hasattr(self.kv_b_proj, "weight") + else self.kv_b_proj.params_dtype + ) + if ( + use_fp8_prefill or kv_b_proj_w_dtype != current_platform.fp8_dtype() + ) and kv_b_proj_w_dtype != torch.uint8: + kv_c_normed = kv_c_normed.to(kv_b_proj_w_dtype) + kv_nope = self.kv_b_proj(kv_c_normed)[0].view( -1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim ) + if use_fp8_prefill: + kv_nope = kv_nope.to(prefill_metadata.q_data_type) + k_pe = k_pe.to(prefill_metadata.q_data_type) k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) k = self._concat_k_nope_k_pe(k_nope, k_pe) @@ -2346,7 +2400,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): q, kv_c_and_k_pe_cache, attn_metadata, - k_scale=None, + k_scale=k_scale, dcp_world_size=self.dcp_world_size, ) ) diff --git a/vllm/v1/attention/backends/mla/flashmla.py b/vllm/v1/attention/backends/mla/flashmla.py index 533e200cac..bb6efe59c8 100644 --- a/vllm/v1/attention/backends/mla/flashmla.py +++ b/vllm/v1/attention/backends/mla/flashmla.py @@ -171,7 +171,10 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]): query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] # we use the max but all should be the same due to uniform length requirement max_query_len = query_lens_cpu.max().item() - num_q_tokens_per_head_k = max_query_len * self.num_q_heads // 1 + num_q_heads = self.num_q_heads + if self.dcp_world_size > 1: + num_q_heads *= self.dcp_world_size + num_q_tokens_per_head_k = max_query_len * num_q_heads // 1 scheduler_metadata, _ = get_mla_metadata( seq_lens_device, num_q_tokens_per_head_k,