[Attention][MLA] FlashMLA sparse: DCP on the fp8_ds_mla mixed-batch path + MTP (#46514)

Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
drakosha
2026-08-19 00:02:09 -04:00
committed by GitHub
co-authored by Claude mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3130573630
commit 63ff748f65
3 changed files with 295 additions and 34 deletions
@@ -1492,6 +1492,8 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b
topk_indices_buffer=topk_indices,
num_heads=2,
kv_lora_rank=1,
dcp_world_size=1,
need_to_return_lse_for_decode=False,
_fp8_flash_mla_kernel=run_kernel,
)
impl._forward_fp8_kv_mixed_batch = MethodType(
@@ -1512,3 +1514,155 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b
assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
assert output.shape == (num_decode_tokens, 2, 1)
assert lse is None
def _build_sparse_dcp_vllm_config(
local_heads: int,
dcp_world_size: int,
comm_backend: str = "ag_rs",
):
"""Minimal sparse-MLA VllmConfig for the FlashMLASparse DCP head-envelope
guard. TP is simulated by mocking ``get_num_attention_heads`` to return the
per-rank head count, as the decode-correctness test above does.
"""
kv_lora_rank = 512
qk_nope_head_dim = 128
qk_rope_head_dim = 64
v_head_dim = 128
head_size = kv_lora_rank + qk_rope_head_dim
topk_tokens = 128
vllm_config = create_vllm_config(
model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
tensor_parallel_size=1,
max_model_len=4096,
block_size=64,
hf_config_override={
"index_topk": topk_tokens,
"attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
},
)
model_config = vllm_config.model_config
model_config.dtype = torch.bfloat16
model_config.hf_text_config = SimpleNamespace(
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: local_heads, model_config
)
model_config.get_num_kv_heads = MethodType(
lambda self, parallel_config: 1, model_config
)
model_config.get_head_size = MethodType(lambda self: head_size, model_config)
model_config.get_sliding_window = MethodType(lambda self: None, model_config)
vllm_config.cache_config.cache_dtype = "fp8_ds_mla"
vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size
vllm_config.parallel_config.dcp_comm_backend = comm_backend
# The base builder clones the layer's dense-MHA prefill backend from
# static_forward_context; the guard tests never run prefill.
vllm_config.compilation_config.static_forward_context["placeholder"] = (
SimpleNamespace(prefill_backend=None)
)
return vllm_config
@pytest.mark.skipif(
torch.cuda.get_device_capability() < (9, 0),
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
@pytest.mark.parametrize(
"local_heads,dcp_world_size,should_raise",
[
(16, 8, True),
(24, 4, True),
(16, 4, False),
(16, 1, False),
],
)
def test_fp8_dcp_head_envelope_guard(local_heads, dcp_world_size, should_raise):
"""The fp8 decode envelope (head padding + tile-scheduler metadata) is
sized from the local head count while the kernel runs on the DCP-gathered
heads, so the builder must reject configs where the two pad differently.
"""
device = torch.device(DEVICE_TYPE)
vllm_config = _build_sparse_dcp_vllm_config(local_heads, dcp_world_size)
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
builder_cls = FlashMLASparseBackend.get_builder_cls()
if should_raise:
with pytest.raises(NotImplementedError, match="envelope"):
builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
else:
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
gathered_heads = local_heads * dcp_world_size
local_pad = 64 if local_heads <= 64 else 128
gathered_pad = 64 if gathered_heads <= 64 else 128
assert builder.fp8_decode_padded_heads == local_pad
assert local_pad == gathered_pad
def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
"""A decode row whose top-k shard holds no local candidates (all -1) has
undefined kernel out/lse; it must come back as (0, -inf), the identity of
the cross-rank LSE merge, or a NaN would survive the merge even at zero
weight (0 * NaN = NaN)."""
num_tokens, num_heads, head_dim = 3, 2, 3
q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
local_indices = torch.tensor(
[[0, 1, -1, -1], [-1, -1, -1, -1], [2, -1, 3, -1]],
dtype=torch.int32,
device=DEVICE_TYPE,
)
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.flashmla_sparse."
"triton_filter_and_convert_dcp_index",
lambda *args, **kwargs: local_indices,
)
def run_kernel(**kwargs):
out = torch.full(
(1, num_tokens, num_heads, 1), float("nan"), device=DEVICE_TYPE
)
lse = torch.full((1, num_heads, num_tokens), float("nan"), device=DEVICE_TYPE)
for token_id in (0, 2): # rows with local candidates get real values
out[0, token_id] = float(token_id + 1)
lse[0, :, token_id] = float(token_id + 1)
return out, lse
metadata = SimpleNamespace(
fp8_extra_metadata=FlashMLASparseMetadata.FP8KernelMetadata(
scheduler_metadata=object(), # type: ignore[arg-type]
dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
),
req_id_per_token=torch.empty(num_tokens, dtype=torch.int32, device=DEVICE_TYPE),
block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
block_size=64,
cp_kv_cache_interleave_size=1,
)
impl = SimpleNamespace(
dcp_world_size=2,
dcp_rank=0,
need_to_return_lse_for_decode=True,
_fp8_flash_mla_kernel=run_kernel,
)
out, lse = FlashMLASparseImpl._forward_fp8_kv_mixed_batch(
impl, q, torch.empty(0, device=DEVICE_TYPE), local_indices, metadata
)
assert torch.equal(out[1], torch.zeros_like(out[1]))
assert torch.isneginf(lse[1]).all()
for token_id in (0, 2):
assert torch.equal(out[token_id], torch.full_like(out[token_id], token_id + 1))
assert torch.equal(lse[token_id], torch.full_like(lse[token_id], token_id + 1))
assert out.is_contiguous()
assert not out.isnan().any()
assert not lse.isnan().any()
@@ -343,4 +343,16 @@ class FlashMLAImpl(MLACommonImpl[FlashMLAMetadata]):
o = reshape_attn_output_for_spec_decode(o)
if self.need_to_return_lse_for_decode:
# FlashMLA returns LSE as [batch, heads, seq_len]; the DCP reducer
# consumes [tokens, heads]. Flattening matters under spec-decode,
# where seq_len > 1. Only DCP consumes lse, so skip the copy
# otherwise.
num_decodes, q_num_heads, seq_len = lse.shape
lse = (
lse.permute(0, 2, 1)
.reshape(num_decodes * seq_len, q_num_heads)
.contiguous()
)
return o, lse
+129 -34
View File
@@ -28,6 +28,7 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.mla.sparse_utils import (
triton_convert_req_index_to_global_index,
triton_filter_and_convert_dcp_index,
)
from vllm.v1.attention.backends.utils import (
reshape_attn_output_for_spec_decode,
@@ -254,9 +255,12 @@ class FlashMLASparseMetadataBuilder(
)
else:
threshold = {16: 128, 32: 128, 64: 256, 128: 256}.get(num_q_heads, 256)
# Varlen decodes are safe under DCP: causality comes from the
# indexer's top-k indices, not from the kernel metadata.
self._init_reorder_batch_threshold(
threshold,
supports_spec_as_decode=True,
supports_dcp_with_varlen=(parallel_config.cp_kv_cache_interleave_size == 1),
)
sm_count = num_compute_units(device.index)
@@ -313,6 +317,42 @@ class FlashMLASparseMetadataBuilder(
device=device,
)
self.fp8_use_mixed_batch = self.num_heads < MIN_HEADS_FOR_BF16_PREFILL
if parallel_config.decode_context_parallel_size > 1:
if parallel_config.dcp_comm_backend != "ag_rs":
raise NotImplementedError(
"DCP for FlashMLA sparse is only validated with the "
"default 'ag_rs' DCP comm backend; got "
f"'{parallel_config.dcp_comm_backend}'"
)
if not self.fp8_use_mixed_batch:
raise NotImplementedError(
"DCP for FlashMLA sparse is only supported on the "
"mixed-batch fp8 path (num_heads < "
f"{MIN_HEADS_FOR_BF16_PREFILL}); the separate "
"prefill/decode path returns the LSE for decode tokens "
"only, while the DCP merge needs it for every token"
)
# Head padding (and the tile-scheduler metadata sized from it) is
# computed from the local head count, but the kernel runs on the
# DCP-gathered heads.
gathered_num_heads = (
self.num_heads * parallel_config.decode_context_parallel_size
)
gathered_padded_heads = FlashMLASparseImpl._compute_fp8_decode_padded_heads(
gathered_num_heads
)
if self.fp8_decode_padded_heads != gathered_padded_heads:
raise NotImplementedError(
"DCP for FlashMLA sparse requires the local and "
"DCP-gathered head counts to pad to the same fp8 decode "
f"kernel envelope; got {self.num_heads} local heads "
f"(pad to {self.fp8_decode_padded_heads}) vs "
f"{gathered_num_heads} gathered heads (pad to "
f"{gathered_padded_heads})"
)
def _build_fp8_mixed_decode_prefill(
self,
common_attn_metadata: CommonAttentionMetadata,
@@ -496,10 +536,9 @@ class FlashMLASparseMetadataBuilder(
) -> FlashMLASparseMetadata:
metadata = super().build(common_prefix_len, common_attn_metadata, fast_build)
fp8_use_mixed_batch = self.num_heads < MIN_HEADS_FOR_BF16_PREFILL
metadata.fp8_use_mixed_batch = fp8_use_mixed_batch
metadata.fp8_use_mixed_batch = self.fp8_use_mixed_batch
if self.use_fp8_kv_cache:
if fp8_use_mixed_batch:
if self.fp8_use_mixed_batch:
metadata.fp8_extra_metadata = self._build_fp8_mixed_decode_prefill(
common_attn_metadata
)
@@ -512,6 +551,8 @@ class FlashMLASparseMetadataBuilder(
class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
can_return_lse_for_decode: bool = True
@staticmethod
def _compute_fp8_decode_padded_heads(num_heads: int) -> int:
# FP8 decode kernel only supports h_q = 64 or 128
@@ -566,6 +607,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
"fp8_ds_mla kv-cache dtype"
)
if self.need_to_return_lse_for_decode and not is_quantized_kv_cache(
kv_cache_dtype
):
raise NotImplementedError(
"DCP for FlashMLA sparse requires an fp8_ds_mla kv-cache; "
"the bf16 sparse path is not supported under DCP."
)
if kv_cache_dtype == "fp8_ds_mla":
# Reserve workspace during initialization
assert vllm_config is not None and vllm_config.model_config is not None
@@ -590,7 +639,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
kv_c_and_k_pe_cache: torch.Tensor,
topk_indices: torch.Tensor,
attn_metadata: FlashMLASparseMetadata,
) -> torch.Tensor:
) -> tuple[torch.Tensor, torch.Tensor]:
# Convert per-request indices to global slots (decode) or workspace
# offsets (prefill). req_id_per_token covers the whole batch; slice it
# to the MQA tokens (q may exclude prefill tokens routed to dense MHA).
@@ -712,7 +761,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
chunk_topk_indices_workspace = topk_indices[chunk.tokens_slice]
chunk_topk_length = topk_length[chunk.tokens_slice]
attn_out[chunk.tokens_slice] = self._bf16_flash_mla_kernel(
attn_out[chunk.tokens_slice], _ = self._bf16_flash_mla_kernel(
chunk_q,
chunk_workspace,
chunk_topk_indices_workspace,
@@ -727,22 +776,42 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
kv_c_and_k_pe_cache: torch.Tensor,
topk_indices: torch.Tensor,
attn_metadata: FlashMLASparseMetadata,
) -> torch.Tensor:
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Mixed batch FP8 forward path that treats all tokens as one batch.
This is equivalent to main branch's approach and avoids the BF16
prefill kernel which has head padding overhead when num_heads is small.
Used when use_mixed_batch is True.
The lse is only returned when DCP needs it, otherwise None.
"""
# Convert per-request indices to global slots (decode) or workspace
# offsets (prefill).
topk_indices = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
)
if self.dcp_world_size > 1:
# The indexer emits global token ids; keep this rank's shard and
# convert to local slots. compact_valid_to_front=False keeps the
# scattered -1s, which the fp8 kernel masks natively and the
# empty-row neutralization below relies on. req_id is sliced to
# topk_indices rows (the converter grids from req_id).
topk_indices = triton_filter_and_convert_dcp_index(
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
attn_metadata.block_table,
topk_indices,
dcp_size=self.dcp_world_size,
dcp_rank=self.dcp_rank,
cp_kv_cache_interleave_size=attn_metadata.cp_kv_cache_interleave_size,
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
compact_valid_to_front=False,
)
else:
# Convert per-request indices to global slots (decode) or workspace
# offsets (prefill).
topk_indices = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
)
assert attn_metadata.fp8_extra_metadata is not None
assert isinstance(
@@ -750,15 +819,29 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
)
fp8_metadata = attn_metadata.fp8_extra_metadata
_attn_out, _ = self._fp8_flash_mla_kernel(
_attn_out, _lse = self._fp8_flash_mla_kernel(
q=q.unsqueeze(0), # unsqueeze to add batch_dim: (T, H, D) -> (1, T, H, D)
kv_c_and_k_pe_cache=kv_c_and_k_pe_cache,
topk_indices=topk_indices.unsqueeze(0), # (T, topk) -> (1, T, topk)
kernel_metadata=fp8_metadata,
)
# Output is (1, T, H, D_v), squeeze back to (T, H, D_v)
return _attn_out.squeeze(0)
out = _attn_out.squeeze(0)
if not self.need_to_return_lse_for_decode:
return out, None
# Kernel LSE is (1, H, T); the DCP merge consumes (T, H).
lse = _lse.squeeze(0).transpose(0, 1)
# Rows where this rank owns none of the selected tokens (all indices
# -1) have undefined out/lse; (0, -inf) is the identity element of the
# cross-rank LSE merge, so it drops this rank from those rows.
empty_rows = (topk_indices == -1).all(dim=-1)
out.masked_fill_(empty_rows.view(-1, 1, 1), 0.0)
lse.masked_fill_(empty_rows.view(-1, 1), float("-inf"))
# The head-padding slice above can leave `out` non-contiguous, and the
# merge feeds it to reduce_scatter.
return out.contiguous(), lse
def _fp8_flash_mla_kernel(
self,
@@ -793,9 +876,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
softmax_scale=self.softmax_scale,
)
# Slice output back to actual head count if we padded
# Slice output and lse back to actual head count if we padded
if actual_num_heads < padded_num_heads:
out = out[:, :, :actual_num_heads, :]
lse = lse[:, :actual_num_heads, :]
return out, lse
@@ -805,35 +889,42 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
kv_c_and_k_pe_cache: torch.Tensor,
topk_indices: torch.Tensor,
topk_length: torch.Tensor | None = None,
) -> torch.Tensor:
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens = q.shape[0]
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(
-1, 1, kv_c_and_k_pe_cache.shape[-1]
)
# NOTE(Chen): kernel requires num_local_head to be a multiple of
# 64 on hopper and 128 on blackwell
if self.num_heads % self.prefill_padding != 0:
assert self.prefill_padding % self.num_heads == 0
# 64 on hopper and 128 on blackwell. Pad from q's head count, not
# self.num_heads: under DCP the heads are all-gathered before this.
actual_num_heads = q.shape[1]
padded_num_heads = (
(actual_num_heads + self.prefill_padding - 1)
// self.prefill_padding
* self.prefill_padding
)
if actual_num_heads < padded_num_heads:
logger.warning_once(
f"Padding num_heads from {self.num_heads} to "
f"{self.prefill_padding} for BF16 sparse prefill kernel"
f"Padding num_heads from {actual_num_heads} to "
f"{padded_num_heads} for BF16 sparse prefill kernel"
)
q_padded = q.new_empty((q.shape[0], self.prefill_padding, q.shape[2]))
q_padded[:, : self.num_heads, :] = q
q_padded = q.new_empty((q.shape[0], padded_num_heads, q.shape[2]))
q_padded[:, :actual_num_heads, :] = q
q = q_padded
topk_indices = topk_indices.view(num_tokens, 1, -1)
output = flash_mla_sparse_fwd(
output, _, lse = flash_mla_sparse_fwd(
q,
kv_c_and_k_pe_cache,
topk_indices,
self.softmax_scale,
topk_length=topk_length,
)[0]
)
output = output[:, : self.num_heads, :]
return output
output = output[:, :actual_num_heads, :]
lse = lse[:, :actual_num_heads]
return output, lse
def forward_mqa(
self,
@@ -859,12 +950,16 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
use_fp8_cache = self.kv_cache_dtype == "fp8_ds_mla"
lse: torch.Tensor | None = None
if not use_fp8_cache:
attn_out = self._forward_bf16_kv(
attn_out, bf16_lse = self._forward_bf16_kv(
q, kv_c_and_k_pe_cache, topk_indices, attn_metadata
)
if self.need_to_return_lse_for_decode:
lse = bf16_lse
elif attn_metadata.fp8_use_mixed_batch:
attn_out = self._forward_fp8_kv_mixed_batch(
attn_out, lse = self._forward_fp8_kv_mixed_batch(
q, kv_c_and_k_pe_cache, topk_indices, attn_metadata
)
else:
@@ -872,4 +967,4 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
q, kv_c_and_k_pe_cache, topk_indices, attn_metadata
)
return attn_out, None
return attn_out, lse