[Attention][Model] FP8 KV cache for Triton DiffKV and MiMo-V2.6-Flash (#58128)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Luca Motz
2026-10-05 21:18:29 -07:00
committed by GitHub
co-authored by Codex Claude Opus 5.5 Lucas Wilkinson
parent 30ab428235
commit 21d93d0d8c
5 changed files with 111 additions and 15 deletions
@@ -35,6 +35,8 @@ NUM_HEADS = [(4, 4), (8, 2), (5, 1)]
HEAD_SIZES = [(128, 128), (192, 128)]
BLOCK_SIZES = [16]
DTYPES = [torch.bfloat16]
# None keeps the KV cache in the query dtype.
KV_CACHE_DTYPES = [None, current_platform.fp8_dtype()]
NUM_BLOCKS = 2048
@@ -80,6 +82,7 @@ def _alloc_segm_buffers(seq_threshold_3D: int, num_query_heads: int, head_size_v
@pytest.mark.parametrize("sliding_window", [None, 128])
@pytest.mark.parametrize("soft_cap", [None, 50.0])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("seq_threshold_3D", SEQ_THRESHOLD_3D_VALUES)
@torch.inference_mode()
def test_triton_unified_attn_diffkv_vs_reference(
@@ -89,13 +92,17 @@ def test_triton_unified_attn_diffkv_vs_reference(
sliding_window: int | None,
soft_cap: float | None,
dtype: torch.dtype,
kv_cache_dtype: torch.dtype | None,
block_size: int,
seq_threshold_3D: int,
quant_query: bool = False,
) -> None:
head_size_qk, head_size_v = head_sizes
# Keep the FA3/FA4 comparison on NVIDIA; ROCm uses the PyTorch oracle.
if not current_platform.is_rocm():
# FA3/FA4 is the reference on NVIDIA; ROCm and the FP8 KV cache (FA would
# also quantize Q) use the PyTorch oracle.
use_torch_ref = current_platform.is_rocm() or kv_cache_dtype is not None
if not use_torch_ref:
fa_version = get_flash_attn_version(
head_size=head_size_qk, head_size_v=head_size_v
)
@@ -126,6 +133,20 @@ def test_triton_unified_attn_diffkv_vs_reference(
)
key_cache = kv_cache[..., :head_size_qk]
value_cache = kv_cache[..., head_size_qk:]
q_descale = k_descale = v_descale = None
if kv_cache_dtype is not None:
# Non-power-of-two scales exercise descale rounding.
k_descale = torch.tensor(0.3, dtype=torch.float32)
v_descale = torch.tensor(0.7, dtype=torch.float32)
kv_cache = torch.cat(
[key_cache / k_descale, value_cache / v_descale], dim=-1
).to(kv_cache_dtype)
key_cache = kv_cache[..., :head_size_qk]
value_cache = kv_cache[..., head_size_qk:]
if quant_query:
q_descale = torch.tensor(0.2, dtype=torch.float32)
query = (query / q_descale).to(kv_cache_dtype)
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
dim=0, dtype=torch.int32
@@ -137,12 +158,20 @@ def test_triton_unified_attn_diffkv_vs_reference(
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
if current_platform.is_rocm():
if use_torch_ref:
ref_key_cache = key_cache.float()
ref_value_cache = value_cache.float()
if kv_cache_dtype is not None:
ref_key_cache *= k_descale
ref_value_cache *= v_descale
# FP32 also keeps the helper's in-place scaling off the kernel input.
ref_query = query.float()
if quant_query:
ref_query *= q_descale
ref_out = ref_paged_attn(
query.float(),
key_cache.float(),
value_cache.float(),
ref_query,
ref_key_cache,
ref_value_cache,
query_lens,
kv_lens,
block_tables,
@@ -200,6 +229,32 @@ def test_triton_unified_attn_diffkv_vs_reference(
softmax_segm_output=segm_output,
softmax_segm_max=segm_max,
softmax_segm_expsum=segm_expsum,
q_descale=q_descale,
k_descale=k_descale,
v_descale=v_descale,
)
torch.testing.assert_close(triton_out, ref_out, atol=2e-2, rtol=2e-2)
@pytest.mark.skipif(
not current_platform.is_cuda() or not current_platform.has_device_capability(89),
reason="FP8 query requires CUDA SM89+",
)
@pytest.mark.parametrize(
"seq_lens,seq_threshold_3D",
[([(129, 463)], 0), ([(1, 2011)], 0), ([(1, 2011)], 8)],
)
def test_triton_unified_attn_diffkv_fp8_query(seq_lens, seq_threshold_3D):
test_triton_unified_attn_diffkv_vs_reference(
seq_lens=seq_lens,
num_heads=(8, 2),
head_sizes=(192, 128),
sliding_window=None,
soft_cap=None,
dtype=torch.bfloat16,
kv_cache_dtype=current_platform.fp8_dtype(),
block_size=16,
seq_threshold_3D=seq_threshold_3D,
quant_query=True,
)