[Attention][MLA] FlashMLA sparse: DCP on the fp8_ds_mla mixed-batch path + MTP (#46514)

Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
drakosha
2026-08-19 00:02:09 -04:00
committed by GitHub
co-authored by Claude mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3130573630
commit 63ff748f65
3 changed files with 295 additions and 34 deletions
@@ -1492,6 +1492,8 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b
topk_indices_buffer=topk_indices,
num_heads=2,
kv_lora_rank=1,
dcp_world_size=1,
need_to_return_lse_for_decode=False,
_fp8_flash_mla_kernel=run_kernel,
)
impl._forward_fp8_kv_mixed_batch = MethodType(
@@ -1512,3 +1514,155 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b
assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
assert output.shape == (num_decode_tokens, 2, 1)
assert lse is None
def _build_sparse_dcp_vllm_config(
local_heads: int,
dcp_world_size: int,
comm_backend: str = "ag_rs",
):
"""Minimal sparse-MLA VllmConfig for the FlashMLASparse DCP head-envelope
guard. TP is simulated by mocking ``get_num_attention_heads`` to return the
per-rank head count, as the decode-correctness test above does.
"""
kv_lora_rank = 512
qk_nope_head_dim = 128
qk_rope_head_dim = 64
v_head_dim = 128
head_size = kv_lora_rank + qk_rope_head_dim
topk_tokens = 128
vllm_config = create_vllm_config(
model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
tensor_parallel_size=1,
max_model_len=4096,
block_size=64,
hf_config_override={
"index_topk": topk_tokens,
"attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
},
)
model_config = vllm_config.model_config
model_config.dtype = torch.bfloat16
model_config.hf_text_config = SimpleNamespace(
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: local_heads, model_config
)
model_config.get_num_kv_heads = MethodType(
lambda self, parallel_config: 1, model_config
)
model_config.get_head_size = MethodType(lambda self: head_size, model_config)
model_config.get_sliding_window = MethodType(lambda self: None, model_config)
vllm_config.cache_config.cache_dtype = "fp8_ds_mla"
vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size
vllm_config.parallel_config.dcp_comm_backend = comm_backend
# The base builder clones the layer's dense-MHA prefill backend from
# static_forward_context; the guard tests never run prefill.
vllm_config.compilation_config.static_forward_context["placeholder"] = (
SimpleNamespace(prefill_backend=None)
)
return vllm_config
@pytest.mark.skipif(
torch.cuda.get_device_capability() < (9, 0),
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
@pytest.mark.parametrize(
"local_heads,dcp_world_size,should_raise",
[
(16, 8, True),
(24, 4, True),
(16, 4, False),
(16, 1, False),
],
)
def test_fp8_dcp_head_envelope_guard(local_heads, dcp_world_size, should_raise):
"""The fp8 decode envelope (head padding + tile-scheduler metadata) is
sized from the local head count while the kernel runs on the DCP-gathered
heads, so the builder must reject configs where the two pad differently.
"""
device = torch.device(DEVICE_TYPE)
vllm_config = _build_sparse_dcp_vllm_config(local_heads, dcp_world_size)
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
builder_cls = FlashMLASparseBackend.get_builder_cls()
if should_raise:
with pytest.raises(NotImplementedError, match="envelope"):
builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
else:
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
gathered_heads = local_heads * dcp_world_size
local_pad = 64 if local_heads <= 64 else 128
gathered_pad = 64 if gathered_heads <= 64 else 128
assert builder.fp8_decode_padded_heads == local_pad
assert local_pad == gathered_pad
def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
"""A decode row whose top-k shard holds no local candidates (all -1) has
undefined kernel out/lse; it must come back as (0, -inf), the identity of
the cross-rank LSE merge, or a NaN would survive the merge even at zero
weight (0 * NaN = NaN)."""
num_tokens, num_heads, head_dim = 3, 2, 3
q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
local_indices = torch.tensor(
[[0, 1, -1, -1], [-1, -1, -1, -1], [2, -1, 3, -1]],
dtype=torch.int32,
device=DEVICE_TYPE,
)
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.flashmla_sparse."
"triton_filter_and_convert_dcp_index",
lambda *args, **kwargs: local_indices,
)
def run_kernel(**kwargs):
out = torch.full(
(1, num_tokens, num_heads, 1), float("nan"), device=DEVICE_TYPE
)
lse = torch.full((1, num_heads, num_tokens), float("nan"), device=DEVICE_TYPE)
for token_id in (0, 2): # rows with local candidates get real values
out[0, token_id] = float(token_id + 1)
lse[0, :, token_id] = float(token_id + 1)
return out, lse
metadata = SimpleNamespace(
fp8_extra_metadata=FlashMLASparseMetadata.FP8KernelMetadata(
scheduler_metadata=object(), # type: ignore[arg-type]
dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
),
req_id_per_token=torch.empty(num_tokens, dtype=torch.int32, device=DEVICE_TYPE),
block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
block_size=64,
cp_kv_cache_interleave_size=1,
)
impl = SimpleNamespace(
dcp_world_size=2,
dcp_rank=0,
need_to_return_lse_for_decode=True,
_fp8_flash_mla_kernel=run_kernel,
)
out, lse = FlashMLASparseImpl._forward_fp8_kv_mixed_batch(
impl, q, torch.empty(0, device=DEVICE_TYPE), local_indices, metadata
)
assert torch.equal(out[1], torch.zeros_like(out[1]))
assert torch.isneginf(lse[1]).all()
for token_id in (0, 2):
assert torch.equal(out[token_id], torch.full_like(out[token_id], token_id + 1))
assert torch.equal(lse[token_id], torch.full_like(lse[token_id], token_id + 1))
assert out.is_contiguous()
assert not out.isnan().any()
assert not lse.isnan().any()