[Bugfix] Use int8 workspace for FlashInfer MLA decode (#48046)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
56da398dac
commit
95d6d6f4bb
@@ -17,6 +17,43 @@ if not current_platform.has_device_capability(100):
|
||||
else:
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
|
||||
# Deepseek R1 MLA config.
|
||||
NUM_HEADS = 128
|
||||
KV_LORA_RANK = 512
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
|
||||
|
||||
|
||||
def _make_decode_inputs(bs: int, block_size: int, dtype: torch.dtype):
|
||||
"""Build valid trtllm MLA decode inputs on the current CUDA device."""
|
||||
max_seq_len_cap = 1024
|
||||
seq_lens = [torch.randint(2, max_seq_len_cap, (1,)).item() for _ in range(bs)]
|
||||
seq_lens[-1] = max_seq_len_cap
|
||||
max_seq_len = max(seq_lens)
|
||||
seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
|
||||
|
||||
# Generate block tables with random but unique block IDs
|
||||
# From https://github.com/flashinfer-ai/flashinfer/pull/1222
|
||||
blocks_per_seq = (seq_lens_tensor + block_size - 1) // block_size
|
||||
max_num_blocks_per_seq = max(blocks_per_seq.max().item(), 4)
|
||||
total_blocks_needed = int(sum(blocks_per_seq))
|
||||
all_block_ids = torch.randperm(total_blocks_needed)
|
||||
|
||||
block_tables = torch.zeros((bs, max_num_blocks_per_seq), dtype=torch.int32)
|
||||
block_id = 0
|
||||
for i in range(bs):
|
||||
num_blocks_needed = blocks_per_seq[i]
|
||||
block_tables[i, :num_blocks_needed] = all_block_ids[
|
||||
block_id : block_id + num_blocks_needed
|
||||
]
|
||||
block_id += num_blocks_needed
|
||||
|
||||
kv_cache = torch.randn(block_tables.numel(), block_size, QK_HEAD_DIM).to(dtype)
|
||||
q = torch.randn(bs, NUM_HEADS, QK_HEAD_DIM).to(dtype)
|
||||
return q, kv_cache, block_tables, seq_lens_tensor, max_seq_len
|
||||
|
||||
|
||||
def ref_mla(
|
||||
out: Tensor, # (bs, num_heads, v_head_dim)
|
||||
@@ -49,49 +86,12 @@ def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(42)
|
||||
|
||||
# Deepseek R1 config
|
||||
num_heads = 128
|
||||
kv_lora_rank = 512
|
||||
qk_nope_head_dim = 128
|
||||
qk_rope_head_dim = 64
|
||||
qk_head_dim = kv_lora_rank + qk_rope_head_dim
|
||||
scale = (qk_nope_head_dim + qk_rope_head_dim) ** -0.5
|
||||
|
||||
MAX_SEQ_LEN = 1024
|
||||
|
||||
seq_lens = [torch.randint(2, MAX_SEQ_LEN, (1,)).item() for _ in range(bs)]
|
||||
seq_lens[-1] = MAX_SEQ_LEN
|
||||
max_seq_len = max(seq_lens)
|
||||
seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
|
||||
|
||||
# Generate block tables with random but unique block IDs
|
||||
# From https://github.com/flashinfer-ai/flashinfer/pull/1222
|
||||
blocks_per_seq = (seq_lens_tensor + block_size - 1) // block_size
|
||||
max_num_blocks_per_seq = max(blocks_per_seq.max().item(), 4)
|
||||
total_blocks_needed = sum(blocks_per_seq)
|
||||
# Get random unique IDs for all blocks
|
||||
all_block_ids = torch.randperm(total_blocks_needed)
|
||||
|
||||
block_id = 0
|
||||
block_tables = torch.zeros(
|
||||
(bs, max_num_blocks_per_seq),
|
||||
dtype=torch.int32,
|
||||
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
|
||||
bs, block_size, dtype
|
||||
)
|
||||
|
||||
# Populate block tables and track block assignments
|
||||
block_id = 0
|
||||
for i in range(bs):
|
||||
num_blocks_needed = blocks_per_seq[i]
|
||||
block_tables[i, :num_blocks_needed] = all_block_ids[
|
||||
block_id : block_id + num_blocks_needed
|
||||
]
|
||||
block_id += num_blocks_needed
|
||||
|
||||
kv_cache = torch.randn(block_tables.numel(), block_size, qk_head_dim).to(dtype)
|
||||
q = torch.randn(bs, num_heads, qk_head_dim).to(dtype)
|
||||
|
||||
out_ref = q.new_zeros(bs, num_heads, kv_lora_rank)
|
||||
ref_mla(out_ref, q, kv_cache, scale, block_tables, seq_lens_tensor)
|
||||
out_ref = q.new_zeros(bs, NUM_HEADS, KV_LORA_RANK)
|
||||
ref_mla(out_ref, q, kv_cache, SCALE, block_tables, seq_lens_tensor)
|
||||
|
||||
workspace_buffer = torch.zeros(
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
@@ -107,13 +107,55 @@ def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
|
||||
query=q,
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace_buffer,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens_tensor,
|
||||
max_seq_len=max_seq_len,
|
||||
bmm1_scale=scale,
|
||||
bmm1_scale=SCALE,
|
||||
)
|
||||
out_ans = out_ans.squeeze(1)
|
||||
torch.testing.assert_close(out_ans, out_ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def test_flashinfer_mla_decode_workspace_supports_autotune():
|
||||
"""vLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
|
||||
|
||||
Model Runner V2's warmup autotunes ``trtllm_batch_decode_mla``, which makes
|
||||
the FlashInfer autotuner enumerate the CuteDSL tactic. That tactic asserts
|
||||
``workspace_buffer.dtype == torch.int8``; the trtllm-gen path (used for
|
||||
normal, non-autotuned inference) instead views the buffer as uint8, so a
|
||||
uint8 workspace only fails once the autotuner tries CuteDSL. That regressed
|
||||
every DeepSeek MLA test on Blackwell under V2 with
|
||||
``workspace_buffer must be torch.int8`` (vllm-project/vllm#46646).
|
||||
"""
|
||||
from flashinfer.autotuner import autotune
|
||||
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla import _get_workspace_buffer
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(0)
|
||||
|
||||
workspace_buffer = _get_workspace_buffer(return_lse=False)
|
||||
assert workspace_buffer.dtype == torch.int8
|
||||
|
||||
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
|
||||
bs=1, block_size=64, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Under the autotuner the CuteDSL tactic is instantiated with our workspace;
|
||||
# a uint8 buffer raises AssertionError here, an int8 buffer succeeds.
|
||||
with torch.inference_mode(), autotune(True):
|
||||
trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q.unsqueeze(1),
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace_buffer,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens_tensor,
|
||||
max_seq_len=max_seq_len,
|
||||
bmm1_scale=SCALE,
|
||||
)
|
||||
|
||||
@@ -42,7 +42,9 @@ def _get_workspace_buffer(return_lse: bool) -> torch.Tensor:
|
||||
else FLASHINFER_MLA_WORKSPACE_BUFFER_SIZE
|
||||
)
|
||||
if _fi_workspace is None or _fi_workspace.numel() < buffer_size:
|
||||
_fi_workspace = torch.zeros(buffer_size, dtype=torch.uint8, device="cuda")
|
||||
# FlashInfer's CuteDSL MLA-decode tactic requires an int8 workspace;
|
||||
# the trtllm-gen path views it as uint8, so int8 is safe for all backends.
|
||||
_fi_workspace = torch.zeros(buffer_size, dtype=torch.int8, device="cuda")
|
||||
return _fi_workspace
|
||||
|
||||
|
||||
|
||||
@@ -355,9 +355,11 @@ _fi_sparse_workspace: torch.Tensor | None = None
|
||||
def _get_workspace_buffer(device: torch.device) -> torch.Tensor:
|
||||
global _fi_sparse_workspace
|
||||
if _fi_sparse_workspace is None:
|
||||
# FlashInfer's CuteDSL MLA-decode tactic requires an int8 workspace;
|
||||
# the trtllm-gen path views it as uint8, so int8 is safe for all backends.
|
||||
_fi_sparse_workspace = torch.zeros(
|
||||
envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.uint8,
|
||||
dtype=torch.int8,
|
||||
device=device,
|
||||
)
|
||||
return _fi_sparse_workspace
|
||||
|
||||
Reference in New Issue
Block a user