[Bugfix][MLA] Reserve sparse prefill buffers before KV cache sizing (#57575)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
Robert Shaw
parent
0390299309
commit
62af3df734
@@ -421,7 +421,18 @@ def test_sparse_backend_decode_correctness(
|
||||
workspace_init,
|
||||
q_scale: float,
|
||||
k_scale: float,
|
||||
monkeypatch,
|
||||
):
|
||||
if (
|
||||
batch_name == "large_q_pure_prefill"
|
||||
and backend_cls == FlashMLASparseBackend
|
||||
and kv_cache_dtype == "fp8_ds_mla"
|
||||
and tensor_parallel_size == 4
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.flashmla_sparse.split_prefill_chunks",
|
||||
lambda rows, capacity: [(i, i + 1) for i in range(len(rows))],
|
||||
)
|
||||
if kv_cache_dtype not in backend_cls.supported_kv_cache_dtypes:
|
||||
pytest.skip(f"{backend_cls.get_name()} does not support {kv_cache_dtype}")
|
||||
|
||||
@@ -774,6 +785,10 @@ def test_sparse_backend_decode_correctness(
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
if backend_cls == FlashMLASparseBackend and kv_cache_dtype == "fp8_ds_mla":
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
current_workspace_manager().lock()
|
||||
backend_output = mock_layer.forward_impl(
|
||||
query_vllm,
|
||||
kv_c_vllm,
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.v1.attention.ops.flashmla import (
|
||||
flash_mla_sparse_fwd,
|
||||
flash_mla_with_kvcache,
|
||||
)
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.deepseek_v2 import Indexer
|
||||
@@ -220,6 +221,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
|
||||
topk_indices: torch.Tensor,
|
||||
topk_length: torch.Tensor | None = None,
|
||||
actual_num_heads: int | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens = q.shape[0]
|
||||
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(
|
||||
@@ -250,7 +252,14 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
|
||||
# Zero (not new_empty) the padded lanes: topk_indices is shared by
|
||||
# all heads, so the kernel reduces across the head group and NaNs
|
||||
# from uninitialized memory would leak into the real heads.
|
||||
q_padded = q.new_zeros((q.shape[0], padded_num_heads, q.shape[2]))
|
||||
if out is None:
|
||||
q_padded = q.new_zeros((q.shape[0], padded_num_heads, q.shape[2]))
|
||||
else:
|
||||
*_, q_padded, _, _ = current_workspace_manager().get_simultaneous(
|
||||
*self.workspace_specs
|
||||
)
|
||||
q_padded = q_padded[:num_tokens]
|
||||
q_padded[:, actual_num_heads:].zero_()
|
||||
q_padded[:, :actual_num_heads, :] = q
|
||||
q = q_padded
|
||||
|
||||
@@ -262,6 +271,7 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
|
||||
self.softmax_scale,
|
||||
attn_sink=attn_sink,
|
||||
topk_length=topk_length,
|
||||
out=out,
|
||||
)
|
||||
|
||||
output = output[:, :actual_num_heads, :]
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import (
|
||||
async_tensor_h2d,
|
||||
@@ -745,6 +746,20 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
self.workspace_specs.append(
|
||||
((prefill_workspace_size, head_size), torch.bfloat16)
|
||||
)
|
||||
prefill_query_heads = num_heads
|
||||
if self.pcp_dcp_kv_gather and self.dcp_world_size > self.pcp_world_size:
|
||||
prefill_query_heads *= parallel_config.tensor_parallel_size
|
||||
padded_prefill_query_heads = round_up(
|
||||
prefill_query_heads, self.prefill_padding
|
||||
)
|
||||
self.workspace_specs.extend(
|
||||
(shape, torch.bfloat16)
|
||||
for shape in (
|
||||
(max_tokens, padded_prefill_query_heads, head_size),
|
||||
(max_tokens, padded_prefill_query_heads, self.kv_lora_rank),
|
||||
(max_tokens, prefill_query_heads, self.kv_lora_rank),
|
||||
)
|
||||
)
|
||||
# Reserve capacity without retaining views that prevent old storage
|
||||
# from being released when another layer grows the shared workspace.
|
||||
current_workspace_manager().get_simultaneous(*self.workspace_specs)
|
||||
@@ -845,7 +860,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
"""All-gather this rank's upconverted KV shard so the chunk's rows attend
|
||||
the whole context, and map their top-k onto the rank-major result."""
|
||||
shard_rows = int(chunk.chunk_tot_seqlen)
|
||||
_, _, gathered_kv_workspace = current_workspace_manager().get_simultaneous(
|
||||
_, _, gathered_kv_workspace, *_ = current_workspace_manager().get_simultaneous(
|
||||
*self.workspace_specs
|
||||
)
|
||||
gathered_kv = gathered_kv_workspace[: self.dcp_world_size * shard_rows]
|
||||
@@ -1007,12 +1022,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
q, decode_topk if decode_topk is not None else topk_indices
|
||||
)
|
||||
else:
|
||||
# Mixed or pure prefill: allocate output tensor
|
||||
attn_out = q.new_empty(
|
||||
(num_mqa_tokens, self.num_heads, self.kv_lora_rank),
|
||||
dtype=q.dtype,
|
||||
device=q.device,
|
||||
_, prefill_bf16_workspace, *_, padded_out, compact_out = (
|
||||
current_workspace_manager().get_simultaneous(*self.workspace_specs)
|
||||
)
|
||||
attn_out = compact_out[:num_mqa_tokens]
|
||||
|
||||
if num_decode_tokens > 0:
|
||||
attn_out[:num_decode_tokens] = _fp8_decode(
|
||||
@@ -1023,9 +1036,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
)
|
||||
|
||||
assert fp8_metadata.prefill is not None
|
||||
_, prefill_bf16_workspace, *_ = (
|
||||
current_workspace_manager().get_simultaneous(*self.workspace_specs)
|
||||
)
|
||||
for chunk_index, chunk in enumerate(fp8_metadata.prefill.chunks):
|
||||
chunk_workspace = prefill_bf16_workspace[: chunk.chunk_tot_seqlen]
|
||||
if uses_host_cache and chunk_index > 0:
|
||||
@@ -1076,12 +1086,15 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
assert topk_length is not None
|
||||
chunk_topk_length = topk_length[chunk.tokens_slice]
|
||||
|
||||
attn_out[chunk.tokens_slice], _ = self._bf16_flash_mla_kernel(
|
||||
chunk_out, chunk_lse = self._bf16_flash_mla_kernel(
|
||||
chunk_q,
|
||||
chunk_workspace,
|
||||
chunk_topk_indices_workspace,
|
||||
chunk_topk_length,
|
||||
out=padded_out[chunk.tokens_slice],
|
||||
)
|
||||
attn_out[chunk.tokens_slice].copy_(chunk_out)
|
||||
del chunk_out, chunk_lse
|
||||
|
||||
if self.pcp_dcp_kv_gather and lse is None:
|
||||
# No decode rows: the DCP merge still expects an LSE, of no rows.
|
||||
@@ -1254,6 +1267,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
topk_indices: torch.Tensor,
|
||||
topk_length: torch.Tensor | None = None,
|
||||
actual_num_heads: int | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens = q.shape[0]
|
||||
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(
|
||||
@@ -1275,7 +1289,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
f"Padding num_heads from {actual_num_heads} to "
|
||||
f"{padded_num_heads} for BF16 sparse prefill kernel"
|
||||
)
|
||||
q_padded = q.new_empty((q.shape[0], padded_num_heads, q.shape[2]))
|
||||
if out is None:
|
||||
q_padded = q.new_empty((q.shape[0], padded_num_heads, q.shape[2]))
|
||||
else:
|
||||
*_, q_padded, _, _ = current_workspace_manager().get_simultaneous(
|
||||
*self.workspace_specs
|
||||
)
|
||||
q_padded = q_padded[:num_tokens]
|
||||
q_padded[:, actual_num_heads:].zero_()
|
||||
q_padded[:, :actual_num_heads, :] = q
|
||||
q = q_padded
|
||||
|
||||
@@ -1286,6 +1307,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
topk_indices,
|
||||
self.softmax_scale,
|
||||
topk_length=topk_length,
|
||||
out=out,
|
||||
)
|
||||
|
||||
output = output[:, :actual_num_heads, :]
|
||||
|
||||
Reference in New Issue
Block a user