[Attention][MLA] FlashMLA sparse: DCP on the fp8_ds_mla mixed-batch path + MTP (#46514)
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3130573630
commit
63ff748f65
@@ -1492,6 +1492,8 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b
|
||||
topk_indices_buffer=topk_indices,
|
||||
num_heads=2,
|
||||
kv_lora_rank=1,
|
||||
dcp_world_size=1,
|
||||
need_to_return_lse_for_decode=False,
|
||||
_fp8_flash_mla_kernel=run_kernel,
|
||||
)
|
||||
impl._forward_fp8_kv_mixed_batch = MethodType(
|
||||
@@ -1512,3 +1514,155 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b
|
||||
assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
|
||||
assert output.shape == (num_decode_tokens, 2, 1)
|
||||
assert lse is None
|
||||
|
||||
|
||||
def _build_sparse_dcp_vllm_config(
|
||||
local_heads: int,
|
||||
dcp_world_size: int,
|
||||
comm_backend: str = "ag_rs",
|
||||
):
|
||||
"""Minimal sparse-MLA VllmConfig for the FlashMLASparse DCP head-envelope
|
||||
guard. TP is simulated by mocking ``get_num_attention_heads`` to return the
|
||||
per-rank head count, as the decode-correctness test above does.
|
||||
"""
|
||||
kv_lora_rank = 512
|
||||
qk_nope_head_dim = 128
|
||||
qk_rope_head_dim = 64
|
||||
v_head_dim = 128
|
||||
head_size = kv_lora_rank + qk_rope_head_dim
|
||||
topk_tokens = 128
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
|
||||
tensor_parallel_size=1,
|
||||
max_model_len=4096,
|
||||
block_size=64,
|
||||
hf_config_override={
|
||||
"index_topk": topk_tokens,
|
||||
"attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
|
||||
},
|
||||
)
|
||||
model_config = vllm_config.model_config
|
||||
model_config.dtype = torch.bfloat16
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: local_heads, model_config
|
||||
)
|
||||
model_config.get_num_kv_heads = MethodType(
|
||||
lambda self, parallel_config: 1, model_config
|
||||
)
|
||||
model_config.get_head_size = MethodType(lambda self: head_size, model_config)
|
||||
model_config.get_sliding_window = MethodType(lambda self: None, model_config)
|
||||
|
||||
vllm_config.cache_config.cache_dtype = "fp8_ds_mla"
|
||||
vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size
|
||||
vllm_config.parallel_config.dcp_comm_backend = comm_backend
|
||||
# The base builder clones the layer's dense-MHA prefill backend from
|
||||
# static_forward_context; the guard tests never run prefill.
|
||||
vllm_config.compilation_config.static_forward_context["placeholder"] = (
|
||||
SimpleNamespace(prefill_backend=None)
|
||||
)
|
||||
return vllm_config
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability() < (9, 0),
|
||||
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"local_heads,dcp_world_size,should_raise",
|
||||
[
|
||||
(16, 8, True),
|
||||
(24, 4, True),
|
||||
(16, 4, False),
|
||||
(16, 1, False),
|
||||
],
|
||||
)
|
||||
def test_fp8_dcp_head_envelope_guard(local_heads, dcp_world_size, should_raise):
|
||||
"""The fp8 decode envelope (head padding + tile-scheduler metadata) is
|
||||
sized from the local head count while the kernel runs on the DCP-gathered
|
||||
heads, so the builder must reject configs where the two pad differently.
|
||||
"""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
vllm_config = _build_sparse_dcp_vllm_config(local_heads, dcp_world_size)
|
||||
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
|
||||
builder_cls = FlashMLASparseBackend.get_builder_cls()
|
||||
|
||||
if should_raise:
|
||||
with pytest.raises(NotImplementedError, match="envelope"):
|
||||
builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
|
||||
else:
|
||||
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
|
||||
gathered_heads = local_heads * dcp_world_size
|
||||
local_pad = 64 if local_heads <= 64 else 128
|
||||
gathered_pad = 64 if gathered_heads <= 64 else 128
|
||||
assert builder.fp8_decode_padded_heads == local_pad
|
||||
assert local_pad == gathered_pad
|
||||
|
||||
|
||||
def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
|
||||
"""A decode row whose top-k shard holds no local candidates (all -1) has
|
||||
undefined kernel out/lse; it must come back as (0, -inf), the identity of
|
||||
the cross-rank LSE merge, or a NaN would survive the merge even at zero
|
||||
weight (0 * NaN = NaN)."""
|
||||
num_tokens, num_heads, head_dim = 3, 2, 3
|
||||
q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
|
||||
local_indices = torch.tensor(
|
||||
[[0, 1, -1, -1], [-1, -1, -1, -1], [2, -1, 3, -1]],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE_TYPE,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.flashmla_sparse."
|
||||
"triton_filter_and_convert_dcp_index",
|
||||
lambda *args, **kwargs: local_indices,
|
||||
)
|
||||
|
||||
def run_kernel(**kwargs):
|
||||
out = torch.full(
|
||||
(1, num_tokens, num_heads, 1), float("nan"), device=DEVICE_TYPE
|
||||
)
|
||||
lse = torch.full((1, num_heads, num_tokens), float("nan"), device=DEVICE_TYPE)
|
||||
for token_id in (0, 2): # rows with local candidates get real values
|
||||
out[0, token_id] = float(token_id + 1)
|
||||
lse[0, :, token_id] = float(token_id + 1)
|
||||
return out, lse
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
fp8_extra_metadata=FlashMLASparseMetadata.FP8KernelMetadata(
|
||||
scheduler_metadata=object(), # type: ignore[arg-type]
|
||||
dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
),
|
||||
req_id_per_token=torch.empty(num_tokens, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
block_size=64,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
)
|
||||
impl = SimpleNamespace(
|
||||
dcp_world_size=2,
|
||||
dcp_rank=0,
|
||||
need_to_return_lse_for_decode=True,
|
||||
_fp8_flash_mla_kernel=run_kernel,
|
||||
)
|
||||
|
||||
out, lse = FlashMLASparseImpl._forward_fp8_kv_mixed_batch(
|
||||
impl, q, torch.empty(0, device=DEVICE_TYPE), local_indices, metadata
|
||||
)
|
||||
|
||||
assert torch.equal(out[1], torch.zeros_like(out[1]))
|
||||
assert torch.isneginf(lse[1]).all()
|
||||
for token_id in (0, 2):
|
||||
assert torch.equal(out[token_id], torch.full_like(out[token_id], token_id + 1))
|
||||
assert torch.equal(lse[token_id], torch.full_like(lse[token_id], token_id + 1))
|
||||
assert out.is_contiguous()
|
||||
assert not out.isnan().any()
|
||||
assert not lse.isnan().any()
|
||||
|
||||
Reference in New Issue
Block a user