[Attention][DCP] Use FlashInfer native CP for MLA decode (#54012)
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -45,7 +45,9 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
|
||||
impl.bmm1_scale = 1.0
|
||||
impl.bmm2_scale = 1.0
|
||||
impl.need_to_return_lse_for_decode = True
|
||||
impl.dcp_world_size = 2
|
||||
impl.dcp_world_size = 1
|
||||
impl.dcp_rank = 0
|
||||
impl.cp_kv_cache_interleave_size = 1
|
||||
impl.num_heads = 6
|
||||
impl.qk_nope_head_dim = 128
|
||||
impl.kv_lora_rank = 512
|
||||
@@ -79,3 +81,64 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
|
||||
assert kernel.call_args.kwargs["backend"] == "cute-dsl"
|
||||
assert output.shape == (2, 24, 512)
|
||||
assert lse is not None and lse.shape == (2, 24)
|
||||
|
||||
|
||||
@requires_flashinfer_mla
|
||||
def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch):
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla as flashinfer_mla
|
||||
|
||||
impl = MagicMock()
|
||||
impl.bmm1_scale = 1.0
|
||||
impl.bmm2_scale = 1.0
|
||||
impl.need_to_return_lse_for_decode = True
|
||||
impl.num_heads = 6
|
||||
impl.qk_nope_head_dim = 128
|
||||
impl.kv_lora_rank = 512
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.dcp_world_size = 8
|
||||
impl.dcp_rank = 3
|
||||
impl.cp_kv_cache_interleave_size = 1
|
||||
|
||||
num_reqs, query_len = 2, 3
|
||||
num_tokens = num_reqs * query_len
|
||||
block_table = torch.tensor([[10], [20]], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([4, 5], dtype=torch.int32)
|
||||
global_causal_seq_lens = torch.tensor([31, 35], dtype=torch.int32)
|
||||
attn_metadata = MagicMock()
|
||||
attn_metadata.causal = True
|
||||
attn_metadata.num_decode_tokens = num_tokens
|
||||
attn_metadata.num_decodes = num_reqs
|
||||
attn_metadata.max_seq_len = 9
|
||||
attn_metadata.decode.block_table = block_table
|
||||
attn_metadata.decode.seq_lens = seq_lens
|
||||
attn_metadata.decode.dcp_tot_seq_lens = global_causal_seq_lens
|
||||
|
||||
query = torch.ones(num_tokens, 24, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.ones(2, 128, 576, dtype=torch.bfloat16)
|
||||
kernel = MagicMock(
|
||||
return_value=(
|
||||
torch.ones(num_reqs, query_len, 24, 512, dtype=torch.bfloat16),
|
||||
torch.ones(num_reqs, query_len, 24, dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_mla, "_get_workspace_buffer", MagicMock())
|
||||
monkeypatch.setattr(flashinfer_mla, "trtllm_batch_decode_with_kv_cache_mla", kernel)
|
||||
|
||||
output, lse = flashinfer_mla.FlashInferMLAImpl.forward_mqa(
|
||||
impl, query, kv_cache, attn_metadata, MagicMock()
|
||||
)
|
||||
|
||||
call = kernel.call_args.kwargs
|
||||
assert call["query"].shape == (num_reqs, query_len, 24, 576)
|
||||
assert call["block_tables"] is block_table
|
||||
assert call["seq_lens"] is seq_lens
|
||||
assert call["backend"] == "cute-dsl"
|
||||
assert call["enable_dcp"] is True
|
||||
assert call["cp_world"] == 8
|
||||
assert call["cp_rank"] == 3
|
||||
assert call["causal_seqlens_kv_global"] is global_causal_seq_lens
|
||||
assert "multi_ctas_kv_counter_buffer" not in call
|
||||
assert flashinfer_mla.FlashInferMLAImpl.lse_base_on_e
|
||||
assert output.shape == (num_tokens, 24, 512)
|
||||
assert lse is not None and lse.shape == (num_tokens, 24)
|
||||
|
||||
@@ -902,7 +902,7 @@ def test_tokenspeed_mla_noncausal_capability():
|
||||
assert tokenspeed_mla_module.TokenspeedMLABackend.supports_non_causal()
|
||||
|
||||
|
||||
def test_flashinfer_mla_dspark_support_is_tp_only(monkeypatch):
|
||||
def test_flashinfer_mla_dspark_dcp_supports_target_only(monkeypatch):
|
||||
flashinfer_mla_module = pytest.importorskip(
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla"
|
||||
)
|
||||
@@ -931,27 +931,11 @@ def test_flashinfer_mla_dspark_support_is_tp_only(monkeypatch):
|
||||
device_capability=SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert reason == "FlashInfer MLA does not support DSpark with DCP"
|
||||
assert reason is None
|
||||
assert backend.supports_non_causal()
|
||||
assert builder.supports_non_causal_multi_token_decode
|
||||
assert not backend.supports_non_causal_dcp()
|
||||
|
||||
vllm_config.parallel_config.decode_context_parallel_size = 1
|
||||
assert (
|
||||
backend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(),
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("causal", "tokens_per_decode", "dcp_world_size", "dcp_rank"),
|
||||
|
||||
Reference in New Issue
Block a user