diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 8ff52ecb38..7704600d09 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -421,7 +421,18 @@ def test_sparse_backend_decode_correctness( workspace_init, q_scale: float, k_scale: float, + monkeypatch, ): + if ( + batch_name == "large_q_pure_prefill" + and backend_cls == FlashMLASparseBackend + and kv_cache_dtype == "fp8_ds_mla" + and tensor_parallel_size == 4 + ): + monkeypatch.setattr( + "vllm.v1.attention.backends.mla.flashmla_sparse.split_prefill_chunks", + lambda rows, capacity: [(i, i + 1) for i in range(len(rows))], + ) if kv_cache_dtype not in backend_cls.supported_kv_cache_dtypes: pytest.skip(f"{backend_cls.get_name()} does not support {kv_cache_dtype}") @@ -774,6 +785,10 @@ def test_sparse_backend_decode_correctness( ) with torch.inference_mode(): + if backend_cls == FlashMLASparseBackend and kv_cache_dtype == "fp8_ds_mla": + from vllm.v1.worker.workspace import current_workspace_manager + + current_workspace_manager().lock() backend_output = mock_layer.forward_impl( query_vllm, kv_c_vllm, diff --git a/vllm/models/hy_v4/nvidia/flashmla_sparse.py b/vllm/models/hy_v4/nvidia/flashmla_sparse.py index 8387175552..1d62f2f556 100644 --- a/vllm/models/hy_v4/nvidia/flashmla_sparse.py +++ b/vllm/models/hy_v4/nvidia/flashmla_sparse.py @@ -34,6 +34,7 @@ from vllm.v1.attention.ops.flashmla import ( flash_mla_sparse_fwd, flash_mla_with_kvcache, ) +from vllm.v1.worker.workspace import current_workspace_manager if TYPE_CHECKING: from vllm.model_executor.models.deepseek_v2 import Indexer @@ -220,6 +221,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl): topk_indices: torch.Tensor, topk_length: torch.Tensor | None = None, actual_num_heads: int | None = None, + out: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: num_tokens = q.shape[0] kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view( @@ -250,7 +252,14 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl): # Zero (not new_empty) the padded lanes: topk_indices is shared by # all heads, so the kernel reduces across the head group and NaNs # from uninitialized memory would leak into the real heads. - q_padded = q.new_zeros((q.shape[0], padded_num_heads, q.shape[2])) + if out is None: + q_padded = q.new_zeros((q.shape[0], padded_num_heads, q.shape[2])) + else: + *_, q_padded, _, _ = current_workspace_manager().get_simultaneous( + *self.workspace_specs + ) + q_padded = q_padded[:num_tokens] + q_padded[:, actual_num_heads:].zero_() q_padded[:, :actual_num_heads, :] = q q = q_padded @@ -262,6 +271,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl): self.softmax_scale, attn_sink=attn_sink, topk_length=topk_length, + out=out, ) output = output[:, :actual_num_heads, :] diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index b3332eb862..aa022066f0 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -19,6 +19,7 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import ( ) from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability +from vllm.utils.math_utils import round_up from vllm.utils.platform_utils import num_compute_units from vllm.utils.torch_utils import ( async_tensor_h2d, @@ -745,6 +746,20 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): self.workspace_specs.append( ((prefill_workspace_size, head_size), torch.bfloat16) ) + prefill_query_heads = num_heads + if self.pcp_dcp_kv_gather and self.dcp_world_size > self.pcp_world_size: + prefill_query_heads *= parallel_config.tensor_parallel_size + padded_prefill_query_heads = round_up( + prefill_query_heads, self.prefill_padding + ) + self.workspace_specs.extend( + (shape, torch.bfloat16) + for shape in ( + (max_tokens, padded_prefill_query_heads, head_size), + (max_tokens, padded_prefill_query_heads, self.kv_lora_rank), + (max_tokens, prefill_query_heads, self.kv_lora_rank), + ) + ) # Reserve capacity without retaining views that prevent old storage # from being released when another layer grows the shared workspace. current_workspace_manager().get_simultaneous(*self.workspace_specs) @@ -845,7 +860,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): """All-gather this rank's upconverted KV shard so the chunk's rows attend the whole context, and map their top-k onto the rank-major result.""" shard_rows = int(chunk.chunk_tot_seqlen) - _, _, gathered_kv_workspace = current_workspace_manager().get_simultaneous( + _, _, gathered_kv_workspace, *_ = current_workspace_manager().get_simultaneous( *self.workspace_specs ) gathered_kv = gathered_kv_workspace[: self.dcp_world_size * shard_rows] @@ -1007,12 +1022,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): q, decode_topk if decode_topk is not None else topk_indices ) else: - # Mixed or pure prefill: allocate output tensor - attn_out = q.new_empty( - (num_mqa_tokens, self.num_heads, self.kv_lora_rank), - dtype=q.dtype, - device=q.device, + _, prefill_bf16_workspace, *_, padded_out, compact_out = ( + current_workspace_manager().get_simultaneous(*self.workspace_specs) ) + attn_out = compact_out[:num_mqa_tokens] if num_decode_tokens > 0: attn_out[:num_decode_tokens] = _fp8_decode( @@ -1023,9 +1036,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): ) assert fp8_metadata.prefill is not None - _, prefill_bf16_workspace, *_ = ( - current_workspace_manager().get_simultaneous(*self.workspace_specs) - ) for chunk_index, chunk in enumerate(fp8_metadata.prefill.chunks): chunk_workspace = prefill_bf16_workspace[: chunk.chunk_tot_seqlen] if uses_host_cache and chunk_index > 0: @@ -1076,12 +1086,15 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): assert topk_length is not None chunk_topk_length = topk_length[chunk.tokens_slice] - attn_out[chunk.tokens_slice], _ = self._bf16_flash_mla_kernel( + chunk_out, chunk_lse = self._bf16_flash_mla_kernel( chunk_q, chunk_workspace, chunk_topk_indices_workspace, chunk_topk_length, + out=padded_out[chunk.tokens_slice], ) + attn_out[chunk.tokens_slice].copy_(chunk_out) + del chunk_out, chunk_lse if self.pcp_dcp_kv_gather and lse is None: # No decode rows: the DCP merge still expects an LSE, of no rows. @@ -1254,6 +1267,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): topk_indices: torch.Tensor, topk_length: torch.Tensor | None = None, actual_num_heads: int | None = None, + out: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: num_tokens = q.shape[0] kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view( @@ -1275,7 +1289,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): f"Padding num_heads from {actual_num_heads} to " f"{padded_num_heads} for BF16 sparse prefill kernel" ) - q_padded = q.new_empty((q.shape[0], padded_num_heads, q.shape[2])) + if out is None: + q_padded = q.new_empty((q.shape[0], padded_num_heads, q.shape[2])) + else: + *_, q_padded, _, _ = current_workspace_manager().get_simultaneous( + *self.workspace_specs + ) + q_padded = q_padded[:num_tokens] + q_padded[:, actual_num_heads:].zero_() q_padded[:, :actual_num_heads, :] = q q = q_padded @@ -1286,6 +1307,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): topk_indices, self.softmax_scale, topk_length=topk_length, + out=out, ) output = output[:, :actual_num_heads, :]