[Bugfix][MLA] Reserve sparse prefill buffers before KV cache sizing (#57575)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Lucas Wilkinson
2026-09-18 21:22:18 +00:00
committed by GitHub
co-authored by Codex Robert Shaw
parent 0390299309
commit 62af3df734
3 changed files with 59 additions and 12 deletions
@@ -421,7 +421,18 @@ def test_sparse_backend_decode_correctness(
workspace_init,
q_scale: float,
k_scale: float,
monkeypatch,
):
if (
batch_name == "large_q_pure_prefill"
and backend_cls == FlashMLASparseBackend
and kv_cache_dtype == "fp8_ds_mla"
and tensor_parallel_size == 4
):
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.flashmla_sparse.split_prefill_chunks",
lambda rows, capacity: [(i, i + 1) for i in range(len(rows))],
)
if kv_cache_dtype not in backend_cls.supported_kv_cache_dtypes:
pytest.skip(f"{backend_cls.get_name()} does not support {kv_cache_dtype}")
@@ -774,6 +785,10 @@ def test_sparse_backend_decode_correctness(
)
with torch.inference_mode():
if backend_cls == FlashMLASparseBackend and kv_cache_dtype == "fp8_ds_mla":
from vllm.v1.worker.workspace import current_workspace_manager
current_workspace_manager().lock()
backend_output = mock_layer.forward_impl(
query_vllm,
kv_c_vllm,
+11 -1
View File
@@ -34,6 +34,7 @@ from vllm.v1.attention.ops.flashmla import (
flash_mla_sparse_fwd,
flash_mla_with_kvcache,
)
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.model_executor.models.deepseek_v2 import Indexer
@@ -220,6 +221,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
topk_indices: torch.Tensor,
topk_length: torch.Tensor | None = None,
actual_num_heads: int | None = None,
out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens = q.shape[0]
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(
@@ -250,7 +252,14 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
# Zero (not new_empty) the padded lanes: topk_indices is shared by
# all heads, so the kernel reduces across the head group and NaNs
# from uninitialized memory would leak into the real heads.
q_padded = q.new_zeros((q.shape[0], padded_num_heads, q.shape[2]))
if out is None:
q_padded = q.new_zeros((q.shape[0], padded_num_heads, q.shape[2]))
else:
*_, q_padded, _, _ = current_workspace_manager().get_simultaneous(
*self.workspace_specs
)
q_padded = q_padded[:num_tokens]
q_padded[:, actual_num_heads:].zero_()
q_padded[:, :actual_num_heads, :] = q
q = q_padded
@@ -262,6 +271,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
self.softmax_scale,
attn_sink=attn_sink,
topk_length=topk_length,
out=out,
)
output = output[:, :actual_num_heads, :]
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
)
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.utils.math_utils import round_up
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import (
async_tensor_h2d,
@@ -745,6 +746,20 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
self.workspace_specs.append(
((prefill_workspace_size, head_size), torch.bfloat16)
)
prefill_query_heads = num_heads
if self.pcp_dcp_kv_gather and self.dcp_world_size > self.pcp_world_size:
prefill_query_heads *= parallel_config.tensor_parallel_size
padded_prefill_query_heads = round_up(
prefill_query_heads, self.prefill_padding
)
self.workspace_specs.extend(
(shape, torch.bfloat16)
for shape in (
(max_tokens, padded_prefill_query_heads, head_size),
(max_tokens, padded_prefill_query_heads, self.kv_lora_rank),
(max_tokens, prefill_query_heads, self.kv_lora_rank),
)
)
# Reserve capacity without retaining views that prevent old storage
# from being released when another layer grows the shared workspace.
current_workspace_manager().get_simultaneous(*self.workspace_specs)
@@ -845,7 +860,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
"""All-gather this rank's upconverted KV shard so the chunk's rows attend
the whole context, and map their top-k onto the rank-major result."""
shard_rows = int(chunk.chunk_tot_seqlen)
_, _, gathered_kv_workspace = current_workspace_manager().get_simultaneous(
_, _, gathered_kv_workspace, *_ = current_workspace_manager().get_simultaneous(
*self.workspace_specs
)
gathered_kv = gathered_kv_workspace[: self.dcp_world_size * shard_rows]
@@ -1007,12 +1022,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
q, decode_topk if decode_topk is not None else topk_indices
)
else:
# Mixed or pure prefill: allocate output tensor
attn_out = q.new_empty(
(num_mqa_tokens, self.num_heads, self.kv_lora_rank),
dtype=q.dtype,
device=q.device,
_, prefill_bf16_workspace, *_, padded_out, compact_out = (
current_workspace_manager().get_simultaneous(*self.workspace_specs)
)
attn_out = compact_out[:num_mqa_tokens]
if num_decode_tokens > 0:
attn_out[:num_decode_tokens] = _fp8_decode(
@@ -1023,9 +1036,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
)
assert fp8_metadata.prefill is not None
_, prefill_bf16_workspace, *_ = (
current_workspace_manager().get_simultaneous(*self.workspace_specs)
)
for chunk_index, chunk in enumerate(fp8_metadata.prefill.chunks):
chunk_workspace = prefill_bf16_workspace[: chunk.chunk_tot_seqlen]
if uses_host_cache and chunk_index > 0:
@@ -1076,12 +1086,15 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
assert topk_length is not None
chunk_topk_length = topk_length[chunk.tokens_slice]
attn_out[chunk.tokens_slice], _ = self._bf16_flash_mla_kernel(
chunk_out, chunk_lse = self._bf16_flash_mla_kernel(
chunk_q,
chunk_workspace,
chunk_topk_indices_workspace,
chunk_topk_length,
out=padded_out[chunk.tokens_slice],
)
attn_out[chunk.tokens_slice].copy_(chunk_out)
del chunk_out, chunk_lse
if self.pcp_dcp_kv_gather and lse is None:
# No decode rows: the DCP merge still expects an LSE, of no rows.
@@ -1254,6 +1267,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
topk_indices: torch.Tensor,
topk_length: torch.Tensor | None = None,
actual_num_heads: int | None = None,
out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens = q.shape[0]
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(
@@ -1275,7 +1289,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
f"Padding num_heads from {actual_num_heads} to "
f"{padded_num_heads} for BF16 sparse prefill kernel"
)
q_padded = q.new_empty((q.shape[0], padded_num_heads, q.shape[2]))
if out is None:
q_padded = q.new_empty((q.shape[0], padded_num_heads, q.shape[2]))
else:
*_, q_padded, _, _ = current_workspace_manager().get_simultaneous(
*self.workspace_specs
)
q_padded = q_padded[:num_tokens]
q_padded[:, actual_num_heads:].zero_()
q_padded[:, :actual_num_heads, :] = q
q = q_padded
@@ -1286,6 +1307,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
topk_indices,
self.softmax_scale,
topk_length=topk_length,
out=out,
)
output = output[:, :actual_num_heads, :]