diff --git a/tests/kernels/attention/test_flashinfer_mla_decode.py b/tests/kernels/attention/test_flashinfer_mla_decode.py index d183f67d39..d1bd55eebe 100644 --- a/tests/kernels/attention/test_flashinfer_mla_decode.py +++ b/tests/kernels/attention/test_flashinfer_mla_decode.py @@ -17,6 +17,43 @@ if not current_platform.has_device_capability(100): else: from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla +# Deepseek R1 MLA config. +NUM_HEADS = 128 +KV_LORA_RANK = 512 +QK_NOPE_HEAD_DIM = 128 +QK_ROPE_HEAD_DIM = 64 +QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM +SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5 + + +def _make_decode_inputs(bs: int, block_size: int, dtype: torch.dtype): + """Build valid trtllm MLA decode inputs on the current CUDA device.""" + max_seq_len_cap = 1024 + seq_lens = [torch.randint(2, max_seq_len_cap, (1,)).item() for _ in range(bs)] + seq_lens[-1] = max_seq_len_cap + max_seq_len = max(seq_lens) + seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32) + + # Generate block tables with random but unique block IDs + # From https://github.com/flashinfer-ai/flashinfer/pull/1222 + blocks_per_seq = (seq_lens_tensor + block_size - 1) // block_size + max_num_blocks_per_seq = max(blocks_per_seq.max().item(), 4) + total_blocks_needed = int(sum(blocks_per_seq)) + all_block_ids = torch.randperm(total_blocks_needed) + + block_tables = torch.zeros((bs, max_num_blocks_per_seq), dtype=torch.int32) + block_id = 0 + for i in range(bs): + num_blocks_needed = blocks_per_seq[i] + block_tables[i, :num_blocks_needed] = all_block_ids[ + block_id : block_id + num_blocks_needed + ] + block_id += num_blocks_needed + + kv_cache = torch.randn(block_tables.numel(), block_size, QK_HEAD_DIM).to(dtype) + q = torch.randn(bs, NUM_HEADS, QK_HEAD_DIM).to(dtype) + return q, kv_cache, block_tables, seq_lens_tensor, max_seq_len + def ref_mla( out: Tensor, # (bs, num_heads, v_head_dim) @@ -49,49 +86,12 @@ def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int): torch.set_default_device("cuda") torch.manual_seed(42) - # Deepseek R1 config - num_heads = 128 - kv_lora_rank = 512 - qk_nope_head_dim = 128 - qk_rope_head_dim = 64 - qk_head_dim = kv_lora_rank + qk_rope_head_dim - scale = (qk_nope_head_dim + qk_rope_head_dim) ** -0.5 - - MAX_SEQ_LEN = 1024 - - seq_lens = [torch.randint(2, MAX_SEQ_LEN, (1,)).item() for _ in range(bs)] - seq_lens[-1] = MAX_SEQ_LEN - max_seq_len = max(seq_lens) - seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32) - - # Generate block tables with random but unique block IDs - # From https://github.com/flashinfer-ai/flashinfer/pull/1222 - blocks_per_seq = (seq_lens_tensor + block_size - 1) // block_size - max_num_blocks_per_seq = max(blocks_per_seq.max().item(), 4) - total_blocks_needed = sum(blocks_per_seq) - # Get random unique IDs for all blocks - all_block_ids = torch.randperm(total_blocks_needed) - - block_id = 0 - block_tables = torch.zeros( - (bs, max_num_blocks_per_seq), - dtype=torch.int32, + q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs( + bs, block_size, dtype ) - # Populate block tables and track block assignments - block_id = 0 - for i in range(bs): - num_blocks_needed = blocks_per_seq[i] - block_tables[i, :num_blocks_needed] = all_block_ids[ - block_id : block_id + num_blocks_needed - ] - block_id += num_blocks_needed - - kv_cache = torch.randn(block_tables.numel(), block_size, qk_head_dim).to(dtype) - q = torch.randn(bs, num_heads, qk_head_dim).to(dtype) - - out_ref = q.new_zeros(bs, num_heads, kv_lora_rank) - ref_mla(out_ref, q, kv_cache, scale, block_tables, seq_lens_tensor) + out_ref = q.new_zeros(bs, NUM_HEADS, KV_LORA_RANK) + ref_mla(out_ref, q, kv_cache, SCALE, block_tables, seq_lens_tensor) workspace_buffer = torch.zeros( FLASHINFER_WORKSPACE_BUFFER_SIZE, @@ -107,13 +107,55 @@ def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int): query=q, kv_cache=kv_cache.unsqueeze(1), workspace_buffer=workspace_buffer, - qk_nope_head_dim=qk_nope_head_dim, - kv_lora_rank=kv_lora_rank, - qk_rope_head_dim=qk_rope_head_dim, + qk_nope_head_dim=QK_NOPE_HEAD_DIM, + kv_lora_rank=KV_LORA_RANK, + qk_rope_head_dim=QK_ROPE_HEAD_DIM, block_tables=block_tables, seq_lens=seq_lens_tensor, max_seq_len=max_seq_len, - bmm1_scale=scale, + bmm1_scale=SCALE, ) out_ans = out_ans.squeeze(1) torch.testing.assert_close(out_ans, out_ref, atol=1e-2, rtol=1e-2) + + +def test_flashinfer_mla_decode_workspace_supports_autotune(): + """vLLM's FlashInfer MLA decode workspace must be int8 for autotuning. + + Model Runner V2's warmup autotunes ``trtllm_batch_decode_mla``, which makes + the FlashInfer autotuner enumerate the CuteDSL tactic. That tactic asserts + ``workspace_buffer.dtype == torch.int8``; the trtllm-gen path (used for + normal, non-autotuned inference) instead views the buffer as uint8, so a + uint8 workspace only fails once the autotuner tries CuteDSL. That regressed + every DeepSeek MLA test on Blackwell under V2 with + ``workspace_buffer must be torch.int8`` (vllm-project/vllm#46646). + """ + from flashinfer.autotuner import autotune + + from vllm.v1.attention.backends.mla.flashinfer_mla import _get_workspace_buffer + + torch.set_default_device("cuda") + torch.manual_seed(0) + + workspace_buffer = _get_workspace_buffer(return_lse=False) + assert workspace_buffer.dtype == torch.int8 + + q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs( + bs=1, block_size=64, dtype=torch.bfloat16 + ) + + # Under the autotuner the CuteDSL tactic is instantiated with our workspace; + # a uint8 buffer raises AssertionError here, an int8 buffer succeeds. + with torch.inference_mode(), autotune(True): + trtllm_batch_decode_with_kv_cache_mla( + query=q.unsqueeze(1), + kv_cache=kv_cache.unsqueeze(1), + workspace_buffer=workspace_buffer, + qk_nope_head_dim=QK_NOPE_HEAD_DIM, + kv_lora_rank=KV_LORA_RANK, + qk_rope_head_dim=QK_ROPE_HEAD_DIM, + block_tables=block_tables, + seq_lens=seq_lens_tensor, + max_seq_len=max_seq_len, + bmm1_scale=SCALE, + ) diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla.py b/vllm/v1/attention/backends/mla/flashinfer_mla.py index 8ebe3c9089..0a6eb14e0c 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla.py @@ -42,7 +42,9 @@ def _get_workspace_buffer(return_lse: bool) -> torch.Tensor: else FLASHINFER_MLA_WORKSPACE_BUFFER_SIZE ) if _fi_workspace is None or _fi_workspace.numel() < buffer_size: - _fi_workspace = torch.zeros(buffer_size, dtype=torch.uint8, device="cuda") + # FlashInfer's CuteDSL MLA-decode tactic requires an int8 workspace; + # the trtllm-gen path views it as uint8, so int8 is safe for all backends. + _fi_workspace = torch.zeros(buffer_size, dtype=torch.int8, device="cuda") return _fi_workspace diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index 97778dfea1..b66952cc74 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -355,9 +355,11 @@ _fi_sparse_workspace: torch.Tensor | None = None def _get_workspace_buffer(device: torch.device) -> torch.Tensor: global _fi_sparse_workspace if _fi_sparse_workspace is None: + # FlashInfer's CuteDSL MLA-decode tactic requires an int8 workspace; + # the trtllm-gen path views it as uint8, so int8 is safe for all backends. _fi_sparse_workspace = torch.zeros( envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE, - dtype=torch.uint8, + dtype=torch.int8, device=device, ) return _fi_sparse_workspace