From 63ff748f657af0e2a95a712fd2cc994f4299ff8c Mon Sep 17 00:00:00 2001 From: drakosha Date: Wed, 19 Aug 2026 07:02:09 +0300 Subject: [PATCH] [Attention][MLA] FlashMLA sparse: DCP on the fp8_ds_mla mixed-batch path + MTP (#46514) Signed-off-by: Mikhail Kostryukov Co-authored-by: Claude Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../v1/attention/test_sparse_mla_backends.py | 154 +++++++++++++++++ vllm/v1/attention/backends/mla/flashmla.py | 12 ++ .../attention/backends/mla/flashmla_sparse.py | 163 ++++++++++++++---- 3 files changed, 295 insertions(+), 34 deletions(-) diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index dec43fd874..ac9a87d08c 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -1492,6 +1492,8 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b topk_indices_buffer=topk_indices, num_heads=2, kv_lora_rank=1, + dcp_world_size=1, + need_to_return_lse_for_decode=False, _fp8_flash_mla_kernel=run_kernel, ) impl._forward_fp8_kv_mixed_batch = MethodType( @@ -1512,3 +1514,155 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)] assert output.shape == (num_decode_tokens, 2, 1) assert lse is None + + +def _build_sparse_dcp_vllm_config( + local_heads: int, + dcp_world_size: int, + comm_backend: str = "ag_rs", +): + """Minimal sparse-MLA VllmConfig for the FlashMLASparse DCP head-envelope + guard. TP is simulated by mocking ``get_num_attention_heads`` to return the + per-rank head count, as the decode-correctness test above does. + """ + kv_lora_rank = 512 + qk_nope_head_dim = 128 + qk_rope_head_dim = 64 + v_head_dim = 128 + head_size = kv_lora_rank + qk_rope_head_dim + topk_tokens = 128 + + vllm_config = create_vllm_config( + model_name="deepseek-ai/DeepSeek-V2-Lite-Chat", + tensor_parallel_size=1, + max_model_len=4096, + block_size=64, + hf_config_override={ + "index_topk": topk_tokens, + "attn_module_list_cfg": [{"topk_tokens": topk_tokens}], + }, + ) + model_config = vllm_config.model_config + model_config.dtype = torch.bfloat16 + model_config.hf_text_config = SimpleNamespace( + q_lora_rank=None, + kv_lora_rank=kv_lora_rank, + qk_nope_head_dim=qk_nope_head_dim, + qk_rope_head_dim=qk_rope_head_dim, + v_head_dim=v_head_dim, + model_type="deepseek_v2", + ) + model_config.get_num_attention_heads = MethodType( + lambda self, parallel_config: local_heads, model_config + ) + model_config.get_num_kv_heads = MethodType( + lambda self, parallel_config: 1, model_config + ) + model_config.get_head_size = MethodType(lambda self: head_size, model_config) + model_config.get_sliding_window = MethodType(lambda self: None, model_config) + + vllm_config.cache_config.cache_dtype = "fp8_ds_mla" + vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size + vllm_config.parallel_config.dcp_comm_backend = comm_backend + # The base builder clones the layer's dense-MHA prefill backend from + # static_forward_context; the guard tests never run prefill. + vllm_config.compilation_config.static_forward_context["placeholder"] = ( + SimpleNamespace(prefill_backend=None) + ) + return vllm_config + + +@pytest.mark.skipif( + torch.cuda.get_device_capability() < (9, 0), + reason="FlashMLASparseBackend requires CUDA 9.0 or higher", +) +@pytest.mark.parametrize( + "local_heads,dcp_world_size,should_raise", + [ + (16, 8, True), + (24, 4, True), + (16, 4, False), + (16, 1, False), + ], +) +def test_fp8_dcp_head_envelope_guard(local_heads, dcp_world_size, should_raise): + """The fp8 decode envelope (head padding + tile-scheduler metadata) is + sized from the local head count while the kernel runs on the DCP-gathered + heads, so the builder must reject configs where the two pad differently. + """ + device = torch.device(DEVICE_TYPE) + vllm_config = _build_sparse_dcp_vllm_config(local_heads, dcp_world_size) + kv_cache_spec = create_standard_kv_cache_spec(vllm_config) + builder_cls = FlashMLASparseBackend.get_builder_cls() + + if should_raise: + with pytest.raises(NotImplementedError, match="envelope"): + builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device) + else: + builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device) + gathered_heads = local_heads * dcp_world_size + local_pad = 64 if local_heads <= 64 else 128 + gathered_pad = 64 if gathered_heads <= 64 else 128 + assert builder.fp8_decode_padded_heads == local_pad + assert local_pad == gathered_pad + + +def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch): + """A decode row whose top-k shard holds no local candidates (all -1) has + undefined kernel out/lse; it must come back as (0, -inf), the identity of + the cross-rank LSE merge, or a NaN would survive the merge even at zero + weight (0 * NaN = NaN).""" + num_tokens, num_heads, head_dim = 3, 2, 3 + q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE) + local_indices = torch.tensor( + [[0, 1, -1, -1], [-1, -1, -1, -1], [2, -1, 3, -1]], + dtype=torch.int32, + device=DEVICE_TYPE, + ) + + monkeypatch.setattr( + "vllm.v1.attention.backends.mla.flashmla_sparse." + "triton_filter_and_convert_dcp_index", + lambda *args, **kwargs: local_indices, + ) + + def run_kernel(**kwargs): + out = torch.full( + (1, num_tokens, num_heads, 1), float("nan"), device=DEVICE_TYPE + ) + lse = torch.full((1, num_heads, num_tokens), float("nan"), device=DEVICE_TYPE) + for token_id in (0, 2): # rows with local candidates get real values + out[0, token_id] = float(token_id + 1) + lse[0, :, token_id] = float(token_id + 1) + return out, lse + + metadata = SimpleNamespace( + fp8_extra_metadata=FlashMLASparseMetadata.FP8KernelMetadata( + scheduler_metadata=object(), # type: ignore[arg-type] + dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE), + cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE), + ), + req_id_per_token=torch.empty(num_tokens, dtype=torch.int32, device=DEVICE_TYPE), + block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE), + block_size=64, + cp_kv_cache_interleave_size=1, + ) + impl = SimpleNamespace( + dcp_world_size=2, + dcp_rank=0, + need_to_return_lse_for_decode=True, + _fp8_flash_mla_kernel=run_kernel, + ) + + out, lse = FlashMLASparseImpl._forward_fp8_kv_mixed_batch( + impl, q, torch.empty(0, device=DEVICE_TYPE), local_indices, metadata + ) + + assert torch.equal(out[1], torch.zeros_like(out[1])) + assert torch.isneginf(lse[1]).all() + for token_id in (0, 2): + assert torch.equal(out[token_id], torch.full_like(out[token_id], token_id + 1)) + assert torch.equal(lse[token_id], torch.full_like(lse[token_id], token_id + 1)) + assert out.is_contiguous() + assert not out.isnan().any() + assert not lse.isnan().any() diff --git a/vllm/v1/attention/backends/mla/flashmla.py b/vllm/v1/attention/backends/mla/flashmla.py index bb6efe59c8..a8c7b2555d 100644 --- a/vllm/v1/attention/backends/mla/flashmla.py +++ b/vllm/v1/attention/backends/mla/flashmla.py @@ -343,4 +343,16 @@ class FlashMLAImpl(MLACommonImpl[FlashMLAMetadata]): o = reshape_attn_output_for_spec_decode(o) + if self.need_to_return_lse_for_decode: + # FlashMLA returns LSE as [batch, heads, seq_len]; the DCP reducer + # consumes [tokens, heads]. Flattening matters under spec-decode, + # where seq_len > 1. Only DCP consumes lse, so skip the copy + # otherwise. + num_decodes, q_num_heads, seq_len = lse.shape + lse = ( + lse.permute(0, 2, 1) + .reshape(num_decodes * seq_len, q_num_heads) + .contiguous() + ) + return o, lse diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index e8be7ad5bb..7b0be952ca 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -28,6 +28,7 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.backends.mla.sparse_utils import ( triton_convert_req_index_to_global_index, + triton_filter_and_convert_dcp_index, ) from vllm.v1.attention.backends.utils import ( reshape_attn_output_for_spec_decode, @@ -254,9 +255,12 @@ class FlashMLASparseMetadataBuilder( ) else: threshold = {16: 128, 32: 128, 64: 256, 128: 256}.get(num_q_heads, 256) + # Varlen decodes are safe under DCP: causality comes from the + # indexer's top-k indices, not from the kernel metadata. self._init_reorder_batch_threshold( threshold, supports_spec_as_decode=True, + supports_dcp_with_varlen=(parallel_config.cp_kv_cache_interleave_size == 1), ) sm_count = num_compute_units(device.index) @@ -313,6 +317,42 @@ class FlashMLASparseMetadataBuilder( device=device, ) + self.fp8_use_mixed_batch = self.num_heads < MIN_HEADS_FOR_BF16_PREFILL + + if parallel_config.decode_context_parallel_size > 1: + if parallel_config.dcp_comm_backend != "ag_rs": + raise NotImplementedError( + "DCP for FlashMLA sparse is only validated with the " + "default 'ag_rs' DCP comm backend; got " + f"'{parallel_config.dcp_comm_backend}'" + ) + if not self.fp8_use_mixed_batch: + raise NotImplementedError( + "DCP for FlashMLA sparse is only supported on the " + "mixed-batch fp8 path (num_heads < " + f"{MIN_HEADS_FOR_BF16_PREFILL}); the separate " + "prefill/decode path returns the LSE for decode tokens " + "only, while the DCP merge needs it for every token" + ) + # Head padding (and the tile-scheduler metadata sized from it) is + # computed from the local head count, but the kernel runs on the + # DCP-gathered heads. + gathered_num_heads = ( + self.num_heads * parallel_config.decode_context_parallel_size + ) + gathered_padded_heads = FlashMLASparseImpl._compute_fp8_decode_padded_heads( + gathered_num_heads + ) + if self.fp8_decode_padded_heads != gathered_padded_heads: + raise NotImplementedError( + "DCP for FlashMLA sparse requires the local and " + "DCP-gathered head counts to pad to the same fp8 decode " + f"kernel envelope; got {self.num_heads} local heads " + f"(pad to {self.fp8_decode_padded_heads}) vs " + f"{gathered_num_heads} gathered heads (pad to " + f"{gathered_padded_heads})" + ) + def _build_fp8_mixed_decode_prefill( self, common_attn_metadata: CommonAttentionMetadata, @@ -496,10 +536,9 @@ class FlashMLASparseMetadataBuilder( ) -> FlashMLASparseMetadata: metadata = super().build(common_prefix_len, common_attn_metadata, fast_build) - fp8_use_mixed_batch = self.num_heads < MIN_HEADS_FOR_BF16_PREFILL - metadata.fp8_use_mixed_batch = fp8_use_mixed_batch + metadata.fp8_use_mixed_batch = self.fp8_use_mixed_batch if self.use_fp8_kv_cache: - if fp8_use_mixed_batch: + if self.fp8_use_mixed_batch: metadata.fp8_extra_metadata = self._build_fp8_mixed_decode_prefill( common_attn_metadata ) @@ -512,6 +551,8 @@ class FlashMLASparseMetadataBuilder( class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): + can_return_lse_for_decode: bool = True + @staticmethod def _compute_fp8_decode_padded_heads(num_heads: int) -> int: # FP8 decode kernel only supports h_q = 64 or 128 @@ -566,6 +607,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): "fp8_ds_mla kv-cache dtype" ) + if self.need_to_return_lse_for_decode and not is_quantized_kv_cache( + kv_cache_dtype + ): + raise NotImplementedError( + "DCP for FlashMLA sparse requires an fp8_ds_mla kv-cache; " + "the bf16 sparse path is not supported under DCP." + ) + if kv_cache_dtype == "fp8_ds_mla": # Reserve workspace during initialization assert vllm_config is not None and vllm_config.model_config is not None @@ -590,7 +639,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): kv_c_and_k_pe_cache: torch.Tensor, topk_indices: torch.Tensor, attn_metadata: FlashMLASparseMetadata, - ) -> torch.Tensor: + ) -> tuple[torch.Tensor, torch.Tensor]: # Convert per-request indices to global slots (decode) or workspace # offsets (prefill). req_id_per_token covers the whole batch; slice it # to the MQA tokens (q may exclude prefill tokens routed to dense MHA). @@ -712,7 +761,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): chunk_topk_indices_workspace = topk_indices[chunk.tokens_slice] chunk_topk_length = topk_length[chunk.tokens_slice] - attn_out[chunk.tokens_slice] = self._bf16_flash_mla_kernel( + attn_out[chunk.tokens_slice], _ = self._bf16_flash_mla_kernel( chunk_q, chunk_workspace, chunk_topk_indices_workspace, @@ -727,22 +776,42 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): kv_c_and_k_pe_cache: torch.Tensor, topk_indices: torch.Tensor, attn_metadata: FlashMLASparseMetadata, - ) -> torch.Tensor: + ) -> tuple[torch.Tensor, torch.Tensor | None]: """Mixed batch FP8 forward path that treats all tokens as one batch. This is equivalent to main branch's approach and avoids the BF16 prefill kernel which has head padding overhead when num_heads is small. Used when use_mixed_batch is True. + + The lse is only returned when DCP needs it, otherwise None. """ - # Convert per-request indices to global slots (decode) or workspace - # offsets (prefill). - topk_indices = triton_convert_req_index_to_global_index( - attn_metadata.req_id_per_token[: topk_indices.shape[0]], - attn_metadata.block_table, - topk_indices, - BLOCK_SIZE=attn_metadata.block_size, - NUM_TOPK_TOKENS=topk_indices.shape[1], - ) + if self.dcp_world_size > 1: + # The indexer emits global token ids; keep this rank's shard and + # convert to local slots. compact_valid_to_front=False keeps the + # scattered -1s, which the fp8 kernel masks natively and the + # empty-row neutralization below relies on. req_id is sliced to + # topk_indices rows (the converter grids from req_id). + topk_indices = triton_filter_and_convert_dcp_index( + attn_metadata.req_id_per_token[: topk_indices.shape[0]], + attn_metadata.block_table, + topk_indices, + dcp_size=self.dcp_world_size, + dcp_rank=self.dcp_rank, + cp_kv_cache_interleave_size=attn_metadata.cp_kv_cache_interleave_size, + BLOCK_SIZE=attn_metadata.block_size, + NUM_TOPK_TOKENS=topk_indices.shape[1], + compact_valid_to_front=False, + ) + else: + # Convert per-request indices to global slots (decode) or workspace + # offsets (prefill). + topk_indices = triton_convert_req_index_to_global_index( + attn_metadata.req_id_per_token[: topk_indices.shape[0]], + attn_metadata.block_table, + topk_indices, + BLOCK_SIZE=attn_metadata.block_size, + NUM_TOPK_TOKENS=topk_indices.shape[1], + ) assert attn_metadata.fp8_extra_metadata is not None assert isinstance( @@ -750,15 +819,29 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): ) fp8_metadata = attn_metadata.fp8_extra_metadata - _attn_out, _ = self._fp8_flash_mla_kernel( + _attn_out, _lse = self._fp8_flash_mla_kernel( q=q.unsqueeze(0), # unsqueeze to add batch_dim: (T, H, D) -> (1, T, H, D) kv_c_and_k_pe_cache=kv_c_and_k_pe_cache, topk_indices=topk_indices.unsqueeze(0), # (T, topk) -> (1, T, topk) kernel_metadata=fp8_metadata, ) - # Output is (1, T, H, D_v), squeeze back to (T, H, D_v) - return _attn_out.squeeze(0) + out = _attn_out.squeeze(0) + + if not self.need_to_return_lse_for_decode: + return out, None + + # Kernel LSE is (1, H, T); the DCP merge consumes (T, H). + lse = _lse.squeeze(0).transpose(0, 1) + # Rows where this rank owns none of the selected tokens (all indices + # -1) have undefined out/lse; (0, -inf) is the identity element of the + # cross-rank LSE merge, so it drops this rank from those rows. + empty_rows = (topk_indices == -1).all(dim=-1) + out.masked_fill_(empty_rows.view(-1, 1, 1), 0.0) + lse.masked_fill_(empty_rows.view(-1, 1), float("-inf")) + # The head-padding slice above can leave `out` non-contiguous, and the + # merge feeds it to reduce_scatter. + return out.contiguous(), lse def _fp8_flash_mla_kernel( self, @@ -793,9 +876,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): softmax_scale=self.softmax_scale, ) - # Slice output back to actual head count if we padded + # Slice output and lse back to actual head count if we padded if actual_num_heads < padded_num_heads: out = out[:, :, :actual_num_heads, :] + lse = lse[:, :actual_num_heads, :] return out, lse @@ -805,35 +889,42 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): kv_c_and_k_pe_cache: torch.Tensor, topk_indices: torch.Tensor, topk_length: torch.Tensor | None = None, - ) -> torch.Tensor: + ) -> tuple[torch.Tensor, torch.Tensor]: num_tokens = q.shape[0] kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view( -1, 1, kv_c_and_k_pe_cache.shape[-1] ) # NOTE(Chen): kernel requires num_local_head to be a multiple of - # 64 on hopper and 128 on blackwell - if self.num_heads % self.prefill_padding != 0: - assert self.prefill_padding % self.num_heads == 0 + # 64 on hopper and 128 on blackwell. Pad from q's head count, not + # self.num_heads: under DCP the heads are all-gathered before this. + actual_num_heads = q.shape[1] + padded_num_heads = ( + (actual_num_heads + self.prefill_padding - 1) + // self.prefill_padding + * self.prefill_padding + ) + if actual_num_heads < padded_num_heads: logger.warning_once( - f"Padding num_heads from {self.num_heads} to " - f"{self.prefill_padding} for BF16 sparse prefill kernel" + f"Padding num_heads from {actual_num_heads} to " + f"{padded_num_heads} for BF16 sparse prefill kernel" ) - q_padded = q.new_empty((q.shape[0], self.prefill_padding, q.shape[2])) - q_padded[:, : self.num_heads, :] = q + q_padded = q.new_empty((q.shape[0], padded_num_heads, q.shape[2])) + q_padded[:, :actual_num_heads, :] = q q = q_padded topk_indices = topk_indices.view(num_tokens, 1, -1) - output = flash_mla_sparse_fwd( + output, _, lse = flash_mla_sparse_fwd( q, kv_c_and_k_pe_cache, topk_indices, self.softmax_scale, topk_length=topk_length, - )[0] + ) - output = output[:, : self.num_heads, :] - return output + output = output[:, :actual_num_heads, :] + lse = lse[:, :actual_num_heads] + return output, lse def forward_mqa( self, @@ -859,12 +950,16 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): use_fp8_cache = self.kv_cache_dtype == "fp8_ds_mla" + lse: torch.Tensor | None = None + if not use_fp8_cache: - attn_out = self._forward_bf16_kv( + attn_out, bf16_lse = self._forward_bf16_kv( q, kv_c_and_k_pe_cache, topk_indices, attn_metadata ) + if self.need_to_return_lse_for_decode: + lse = bf16_lse elif attn_metadata.fp8_use_mixed_batch: - attn_out = self._forward_fp8_kv_mixed_batch( + attn_out, lse = self._forward_fp8_kv_mixed_batch( q, kv_c_and_k_pe_cache, topk_indices, attn_metadata ) else: @@ -872,4 +967,4 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): q, kv_c_and_k_pe_cache, topk_indices, attn_metadata ) - return attn_out, None + return attn_out, lse