Update FlashMLA (#32491)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-01-21 13:03:37 +08:00
committed by GitHub
parent 7ab80a8e37
commit b4f64e5b02
4 changed files with 169 additions and 42 deletions
@@ -43,7 +43,7 @@ def test_sparse_flashmla_decode_smoke():
device = torch.device("cuda")
batch_size = 1
seqlen_q = 1
num_heads_q = 1
num_heads_q = 64
head_dim_k = 576
head_dim_v = 512
num_heads_k = 1
+64 -11
View File
@@ -51,10 +51,34 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
)
def _float_to_e8m0_truncate(f: float) -> float:
"""Simulate SM100's float -> e8m0 -> bf16 scale conversion.
e8m0 format only stores the exponent (power of 2).
cudaRoundZero truncates toward zero, meaning we round down to the
nearest power of 2.
"""
if f <= 0:
return 0.0
# e8m0 = floor(log2(f)), then 2^(e8m0)
# This is equivalent to truncating to the nearest power of 2 below f
exp = math.floor(math.log2(f))
return 2.0**exp
def _dequantize_fp8_ds_mla_entry(
cache_slice: torch.Tensor, kv_lora_rank: int, rope_dim: int, dtype: torch.dtype
cache_slice: torch.Tensor,
kv_lora_rank: int,
rope_dim: int,
dtype: torch.dtype,
simulate_sm100_e8m0_scales: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Dequantize a single fp8_ds_mla cache entry back to latent + rope."""
"""Dequantize a single fp8_ds_mla cache entry back to latent + rope.
Args:
simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
float -> e8m0 -> bf16 scale conversion path.
"""
# The first kv_lora_rank bytes store FP8 latent values with one scale per
# 128 element tile written as float32 right after the latent payload.
@@ -63,10 +87,14 @@ def _dequantize_fp8_ds_mla_entry(
for tile_idx in range(4):
tile_start = tile_idx * 128
tile_end = tile_start + 128
scale_val = float(scales[tile_idx].item())
if simulate_sm100_e8m0_scales:
# Simulate the lossy float -> e8m0 -> bf16 conversion
scale_val = _float_to_e8m0_truncate(scale_val)
ops.convert_fp8(
latent[tile_start:tile_end],
cache_slice[tile_start:tile_end],
float(scales[tile_idx].item()),
scale_val,
kv_dtype="fp8",
)
latent = latent.to(dtype)
@@ -77,9 +105,18 @@ def _dequantize_fp8_ds_mla_entry(
def _quantize_dequantize_fp8_ds_mla(
kv_c: torch.Tensor, k_pe: torch.Tensor, block_size: int, scale: torch.Tensor
kv_c: torch.Tensor,
k_pe: torch.Tensor,
block_size: int,
scale: torch.Tensor,
simulate_sm100_e8m0_scales: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Round-trip kv_c/k_pe though the fp8_ds_mla cache layout."""
"""Round-trip kv_c/k_pe though the fp8_ds_mla cache layout.
Args:
simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
float -> e8m0 -> bf16 scale conversion in dequantization.
"""
if kv_c.numel() == 0:
return kv_c.clone(), k_pe.clone()
@@ -108,7 +145,11 @@ def _quantize_dequantize_fp8_ds_mla(
block_offset = slot % block_size
cache_slice = tmp_cache[block_idx, block_offset]
latent, rope_vals = _dequantize_fp8_ds_mla_entry(
cache_slice, kv_lora_rank, rope_dim, kv_c.dtype
cache_slice,
kv_lora_rank,
rope_dim,
kv_c.dtype,
simulate_sm100_e8m0_scales=simulate_sm100_e8m0_scales,
)
dequant_kv_c[token_idx] = latent
dequant_k_pe[token_idx] = rope_vals
@@ -143,7 +184,10 @@ def test_sparse_backend_decode_correctness(
batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name]
# Model hyper-parameters (kept intentionally small for the unit test)
num_heads = 128
total_num_heads = 128
# Compute per-rank heads for simulated TP
num_heads = max(1, total_num_heads // tensor_parallel_size)
kv_lora_rank = 512
qk_nope_head_dim = 128
qk_rope_head_dim = 64
@@ -179,7 +223,7 @@ def test_sparse_backend_decode_correctness(
)
model_config.dtype = dtype
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: max(1, num_heads // tensor_parallel_size),
lambda self, parallel_config: num_heads,
model_config,
)
model_config.get_num_kv_heads = MethodType(
@@ -195,10 +239,10 @@ def test_sparse_backend_decode_correctness(
scale = 1.0 / math.sqrt(head_size)
# Shared MLA projection weights to keep reference and backend in sync
W_UK = torch.randn(
W_UK = torch.rand(
kv_lora_rank, num_heads, qk_nope_head_dim, dtype=dtype, device=device
)
W_UV = torch.randn(kv_lora_rank, num_heads, v_head_dim, dtype=dtype, device=device)
W_UV = torch.rand(kv_lora_rank, num_heads, v_head_dim, dtype=dtype, device=device)
# Build synthetic decode-only workload
seq_lens = batch_spec.seq_lens
@@ -225,11 +269,15 @@ def test_sparse_backend_decode_correctness(
kv_c_full = torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device)
k_pe_full = torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)
# SM100 (Blackwell) uses float -> e8m0 -> bf16 scale conversion
# which truncates scales to powers of 2. Simulate this in reference.
is_sm100 = torch.cuda.get_device_capability()[0] >= 10
kv_c_full, k_pe_full = _quantize_dequantize_fp8_ds_mla(
kv_c_full,
k_pe_full.squeeze(1),
block_size=vllm_config.cache_config.block_size,
scale=kv_cache_scale,
simulate_sm100_e8m0_scales=is_sm100,
)
q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
@@ -381,7 +429,12 @@ def test_sparse_backend_decode_correctness(
assert backend_output.dtype == sdpa_reference.dtype
assert torch.isfinite(backend_output).all()
torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.5, atol=0.5)
# FP8 quantization introduces some error, but should be within reasonable bounds
# BF16 (auto) should be very accurate, FP8 allows slightly more tolerance
if kv_cache_dtype == "fp8_ds_mla":
torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.05, atol=0.05)
else:
torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01)
def _triton_convert_reference_impl(