[Attention][Model] FP8 KV cache for Triton DiffKV and MiMo-V2.6-Flash (#58128)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Luca Motz
2026-10-05 21:18:29 -07:00
committed by GitHub
co-authored by Codex Claude Opus 5.5 Lucas Wilkinson
parent 30ab428235
commit 21d93d0d8c
5 changed files with 111 additions and 15 deletions
@@ -35,6 +35,8 @@ NUM_HEADS = [(4, 4), (8, 2), (5, 1)]
HEAD_SIZES = [(128, 128), (192, 128)]
BLOCK_SIZES = [16]
DTYPES = [torch.bfloat16]
# None keeps the KV cache in the query dtype.
KV_CACHE_DTYPES = [None, current_platform.fp8_dtype()]
NUM_BLOCKS = 2048
@@ -80,6 +82,7 @@ def _alloc_segm_buffers(seq_threshold_3D: int, num_query_heads: int, head_size_v
@pytest.mark.parametrize("sliding_window", [None, 128])
@pytest.mark.parametrize("soft_cap", [None, 50.0])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("seq_threshold_3D", SEQ_THRESHOLD_3D_VALUES)
@torch.inference_mode()
def test_triton_unified_attn_diffkv_vs_reference(
@@ -89,13 +92,17 @@ def test_triton_unified_attn_diffkv_vs_reference(
sliding_window: int | None,
soft_cap: float | None,
dtype: torch.dtype,
kv_cache_dtype: torch.dtype | None,
block_size: int,
seq_threshold_3D: int,
quant_query: bool = False,
) -> None:
head_size_qk, head_size_v = head_sizes
# Keep the FA3/FA4 comparison on NVIDIA; ROCm uses the PyTorch oracle.
if not current_platform.is_rocm():
# FA3/FA4 is the reference on NVIDIA; ROCm and the FP8 KV cache (FA would
# also quantize Q) use the PyTorch oracle.
use_torch_ref = current_platform.is_rocm() or kv_cache_dtype is not None
if not use_torch_ref:
fa_version = get_flash_attn_version(
head_size=head_size_qk, head_size_v=head_size_v
)
@@ -126,6 +133,20 @@ def test_triton_unified_attn_diffkv_vs_reference(
)
key_cache = kv_cache[..., :head_size_qk]
value_cache = kv_cache[..., head_size_qk:]
q_descale = k_descale = v_descale = None
if kv_cache_dtype is not None:
# Non-power-of-two scales exercise descale rounding.
k_descale = torch.tensor(0.3, dtype=torch.float32)
v_descale = torch.tensor(0.7, dtype=torch.float32)
kv_cache = torch.cat(
[key_cache / k_descale, value_cache / v_descale], dim=-1
).to(kv_cache_dtype)
key_cache = kv_cache[..., :head_size_qk]
value_cache = kv_cache[..., head_size_qk:]
if quant_query:
q_descale = torch.tensor(0.2, dtype=torch.float32)
query = (query / q_descale).to(kv_cache_dtype)
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
dim=0, dtype=torch.int32
@@ -137,12 +158,20 @@ def test_triton_unified_attn_diffkv_vs_reference(
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
if current_platform.is_rocm():
if use_torch_ref:
ref_key_cache = key_cache.float()
ref_value_cache = value_cache.float()
if kv_cache_dtype is not None:
ref_key_cache *= k_descale
ref_value_cache *= v_descale
# FP32 also keeps the helper's in-place scaling off the kernel input.
ref_query = query.float()
if quant_query:
ref_query *= q_descale
ref_out = ref_paged_attn(
query.float(),
key_cache.float(),
value_cache.float(),
ref_query,
ref_key_cache,
ref_value_cache,
query_lens,
kv_lens,
block_tables,
@@ -200,6 +229,32 @@ def test_triton_unified_attn_diffkv_vs_reference(
softmax_segm_output=segm_output,
softmax_segm_max=segm_max,
softmax_segm_expsum=segm_expsum,
q_descale=q_descale,
k_descale=k_descale,
v_descale=v_descale,
)
torch.testing.assert_close(triton_out, ref_out, atol=2e-2, rtol=2e-2)
@pytest.mark.skipif(
not current_platform.is_cuda() or not current_platform.has_device_capability(89),
reason="FP8 query requires CUDA SM89+",
)
@pytest.mark.parametrize(
"seq_lens,seq_threshold_3D",
[([(129, 463)], 0), ([(1, 2011)], 0), ([(1, 2011)], 8)],
)
def test_triton_unified_attn_diffkv_fp8_query(seq_lens, seq_threshold_3D):
test_triton_unified_attn_diffkv_vs_reference(
seq_lens=seq_lens,
num_heads=(8, 2),
head_sizes=(192, 128),
sliding_window=None,
soft_cap=None,
dtype=torch.bfloat16,
kv_cache_dtype=current_platform.fp8_dtype(),
block_size=16,
seq_threshold_3D=seq_threshold_3D,
quant_query=True,
)
+6
View File
@@ -372,6 +372,10 @@ class MiMoV2FlashDecoderLayer(nn.Module):
max_position_embeddings = getattr(config, "max_position_embeddings", 32768)
v_scale = getattr(config, "attention_value_scale", None)
# cache_config.sliding_window holds MiMo's SWA window; global layers
# would otherwise fall back to it in Attention.
cache_config = vllm_config.cache_config
cache_config.sliding_window = None
if self.is_compressed_softmax_layer():
self.self_attn = MiMoV2Attention(
@@ -389,6 +393,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
layer_id=layer_id,
rope_theta=getattr(config, "swa_rope_theta", rope_theta),
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
@@ -406,6 +411,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
layer_id=layer_id,
rope_theta=rope_theta,
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
+6 -1
View File
@@ -24,7 +24,7 @@ import torch
import torch.nn as nn
from transformers import PreTrainedConfig
from vllm.config import VllmConfig
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -64,6 +64,7 @@ class MiMoV2MTPLayer(nn.Module):
self,
config: PreTrainedConfig,
prefix: str,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
@@ -100,6 +101,7 @@ class MiMoV2MTPLayer(nn.Module):
layer_id=0,
rope_theta=swa_rope_theta,
max_position_embeddings=getattr(config, "max_position_embeddings", 32768),
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
@@ -147,6 +149,7 @@ class _MiMoV2MTPLayers(nn.Module):
self,
config: PreTrainedConfig,
num_mtp_layers: int,
cache_config: CacheConfig | None,
quant_config: QuantizationConfig | None,
prefix: str,
) -> None:
@@ -156,6 +159,7 @@ class _MiMoV2MTPLayers(nn.Module):
str(i): MiMoV2MTPLayer(
config=config,
prefix=f"{prefix}.{i}",
cache_config=cache_config,
quant_config=quant_config,
)
for i in range(num_mtp_layers)
@@ -182,6 +186,7 @@ class MiMoV2MultiTokenPredictor(nn.Module):
self.mtp = _MiMoV2MTPLayers(
config=config,
num_mtp_layers=num_mtp_layers,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "mtp.layers"),
)
@@ -70,10 +70,11 @@ class TritonAttentionDiffKVBackend(TritonAttentionBackend):
# V head dim — set per layer via ``set_head_size_v`` before instantiation.
head_size_v: int = 128
# No FP8 / int8 KV cache for the DiffKV path yet; require fp16/bf16/fp32.
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
"bfloat16",
"fp8",
"fp8_e4m3",
]
@classmethod
@@ -110,16 +111,15 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
if is_quantized_kv_cache(self.kv_cache_dtype):
if (
is_quantized_kv_cache(self.kv_cache_dtype)
and self.kv_cache_dtype
not in TritonAttentionDiffKVBackend.supported_kv_cache_dtypes
):
raise NotImplementedError(
"TritonAttentionDiffKVBackend does not yet support quantized "
"TritonAttentionDiffKVBackend supports only fp8/fp8_e4m3 quantized "
f"KV cache (got kv_cache_dtype={self.kv_cache_dtype!r})."
)
if self._is_per_token_head_quant:
raise NotImplementedError(
"TritonAttentionDiffKVBackend does not support per-token-head "
"quantization."
)
if self.chunk_lookback > -1:
raise NotImplementedError(
"TritonAttentionDiffKVBackend does not support chunked "
@@ -191,6 +191,9 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
# Triton DiffKV kernels consume (B, N, H, D) cache views.
kv_cache = kv_cache.transpose(1, 2)
quantized = is_quantized_kv_cache(self.kv_cache_dtype)
if quantized:
kv_cache = kv_cache.view(self.fp8_dtype)
key_cache = kv_cache[..., :head_size_qk]
value_cache = kv_cache[..., head_size_qk : head_size_qk + head_size_v]
@@ -202,6 +205,11 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
cu_seqlens_q=attn_metadata.query_start_loc,
seqused_k=attn_metadata.seq_lens,
softmax_scale=self.scale,
q_descale=(
layer._q_scale if quantized and query.dtype == self.fp8_dtype else None
),
k_descale=layer._k_scale if quantized else None,
v_descale=layer._v_scale if quantized else None,
causal=True,
alibi_slopes=self.alibi_slopes,
use_alibi_sqrt=self.use_alibi_sqrt,
@@ -60,6 +60,9 @@ def kernel_unified_attention_diffkv(
query_ptr,
key_cache_ptr, # view of packed cache: [..., :head_size_qk]
value_cache_ptr, # view of packed cache: [..., head_size_qk:hqk+hv]
q_descale_ptr,
k_descale_ptr,
v_descale_ptr,
sink_ptr,
block_tables_ptr,
seq_lens_ptr,
@@ -83,6 +86,8 @@ def kernel_unified_attention_diffkv(
USE_ALIBI_SQRT: tl.constexpr,
USE_SOFTCAP: tl.constexpr,
USE_SINKS: tl.constexpr,
USE_Q_SCALE: tl.constexpr,
USE_KV_SCALES: tl.constexpr,
SLIDING_WINDOW: tl.constexpr,
# Strides for both cache views (they share the same packed buffer, so
# dims 0/1/2 strides match; only the per-head extent differs).
@@ -154,6 +159,12 @@ def kernel_unified_attention_diffkv(
other=0.0,
)
if USE_Q_SCALE:
scale *= tl.load(q_descale_ptr)
if USE_KV_SCALES:
scale *= tl.load(k_descale_ptr)
v_descale = tl.load(v_descale_ptr)
block_table_offset = seq_idx * block_table_stride
M = init_softmax_M(
@@ -262,6 +273,9 @@ def kernel_unified_attention_diffkv(
acc += tl.dot(P.to(V.dtype), V)
# ---- Epilogue --------------------------------------------------------
if USE_KV_SCALES:
acc *= v_descale
if IS_3D:
# Store per-segment partials; finalized by reduce_segments_diffkv.
segm_output_offset = (
@@ -403,6 +417,9 @@ def unified_attention_diffkv(
softmax_segm_output: torch.Tensor | None = None,
softmax_segm_max: torch.Tensor | None = None,
softmax_segm_expsum: torch.Tensor | None = None,
k_descale: torch.Tensor | None = None,
v_descale: torch.Tensor | None = None,
q_descale: torch.Tensor | None = None,
):
assert causal, "Only causal attention is supported"
@@ -470,6 +487,9 @@ def unified_attention_diffkv(
query_ptr=q,
key_cache_ptr=k,
value_cache_ptr=v,
q_descale_ptr=q_descale,
k_descale_ptr=k_descale,
v_descale_ptr=v_descale,
sink_ptr=sinks,
block_tables_ptr=block_table,
seq_lens_ptr=seqused_k,
@@ -493,6 +513,8 @@ def unified_attention_diffkv(
USE_ALIBI_SQRT=use_alibi_sqrt,
USE_SOFTCAP=(softcap > 0),
USE_SINKS=(sinks is not None),
USE_Q_SCALE=q_descale is not None,
USE_KV_SCALES=k_descale is not None,
SLIDING_WINDOW=sliding_window_val,
stride_k_cache_0=k.stride(0),
stride_k_cache_1=k.stride(1),