From 21d93d0d8c0e9627900020382bfce4730e61cab7 Mon Sep 17 00:00:00 2001 From: Luca Motz Date: Tue, 6 Oct 2026 06:18:29 +0200 Subject: [PATCH] [Attention][Model] FP8 KV cache for Triton DiffKV and MiMo-V2.6-Flash (#58128) Signed-off-by: Luca Motz Co-authored-by: Codex Co-authored-by: Claude Opus 5.5 Co-authored-by: Lucas Wilkinson --- .../test_triton_unified_attention_diffkv.py | 67 +++++++++++++++++-- vllm/model_executor/models/mimo_v2.py | 6 ++ vllm/model_executor/models/mimo_v2_mtp.py | 7 +- .../attention/backends/triton_attn_diffkv.py | 24 ++++--- .../ops/triton_unified_attention_diffkv.py | 22 ++++++ 5 files changed, 111 insertions(+), 15 deletions(-) diff --git a/tests/kernels/attention/test_triton_unified_attention_diffkv.py b/tests/kernels/attention/test_triton_unified_attention_diffkv.py index 213d896c03..3b6cf524e2 100644 --- a/tests/kernels/attention/test_triton_unified_attention_diffkv.py +++ b/tests/kernels/attention/test_triton_unified_attention_diffkv.py @@ -35,6 +35,8 @@ NUM_HEADS = [(4, 4), (8, 2), (5, 1)] HEAD_SIZES = [(128, 128), (192, 128)] BLOCK_SIZES = [16] DTYPES = [torch.bfloat16] +# None keeps the KV cache in the query dtype. +KV_CACHE_DTYPES = [None, current_platform.fp8_dtype()] NUM_BLOCKS = 2048 @@ -80,6 +82,7 @@ def _alloc_segm_buffers(seq_threshold_3D: int, num_query_heads: int, head_size_v @pytest.mark.parametrize("sliding_window", [None, 128]) @pytest.mark.parametrize("soft_cap", [None, 50.0]) @pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES) @pytest.mark.parametrize("seq_threshold_3D", SEQ_THRESHOLD_3D_VALUES) @torch.inference_mode() def test_triton_unified_attn_diffkv_vs_reference( @@ -89,13 +92,17 @@ def test_triton_unified_attn_diffkv_vs_reference( sliding_window: int | None, soft_cap: float | None, dtype: torch.dtype, + kv_cache_dtype: torch.dtype | None, block_size: int, seq_threshold_3D: int, + quant_query: bool = False, ) -> None: head_size_qk, head_size_v = head_sizes - # Keep the FA3/FA4 comparison on NVIDIA; ROCm uses the PyTorch oracle. - if not current_platform.is_rocm(): + # FA3/FA4 is the reference on NVIDIA; ROCm and the FP8 KV cache (FA would + # also quantize Q) use the PyTorch oracle. + use_torch_ref = current_platform.is_rocm() or kv_cache_dtype is not None + if not use_torch_ref: fa_version = get_flash_attn_version( head_size=head_size_qk, head_size_v=head_size_v ) @@ -126,6 +133,20 @@ def test_triton_unified_attn_diffkv_vs_reference( ) key_cache = kv_cache[..., :head_size_qk] value_cache = kv_cache[..., head_size_qk:] + q_descale = k_descale = v_descale = None + if kv_cache_dtype is not None: + # Non-power-of-two scales exercise descale rounding. + k_descale = torch.tensor(0.3, dtype=torch.float32) + v_descale = torch.tensor(0.7, dtype=torch.float32) + kv_cache = torch.cat( + [key_cache / k_descale, value_cache / v_descale], dim=-1 + ).to(kv_cache_dtype) + key_cache = kv_cache[..., :head_size_qk] + value_cache = kv_cache[..., head_size_qk:] + + if quant_query: + q_descale = torch.tensor(0.2, dtype=torch.float32) + query = (query / q_descale).to(kv_cache_dtype) cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum( dim=0, dtype=torch.int32 @@ -137,12 +158,20 @@ def test_triton_unified_attn_diffkv_vs_reference( 0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32 ) - if current_platform.is_rocm(): + if use_torch_ref: + ref_key_cache = key_cache.float() + ref_value_cache = value_cache.float() + if kv_cache_dtype is not None: + ref_key_cache *= k_descale + ref_value_cache *= v_descale # FP32 also keeps the helper's in-place scaling off the kernel input. + ref_query = query.float() + if quant_query: + ref_query *= q_descale ref_out = ref_paged_attn( - query.float(), - key_cache.float(), - value_cache.float(), + ref_query, + ref_key_cache, + ref_value_cache, query_lens, kv_lens, block_tables, @@ -200,6 +229,32 @@ def test_triton_unified_attn_diffkv_vs_reference( softmax_segm_output=segm_output, softmax_segm_max=segm_max, softmax_segm_expsum=segm_expsum, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, ) torch.testing.assert_close(triton_out, ref_out, atol=2e-2, rtol=2e-2) + + +@pytest.mark.skipif( + not current_platform.is_cuda() or not current_platform.has_device_capability(89), + reason="FP8 query requires CUDA SM89+", +) +@pytest.mark.parametrize( + "seq_lens,seq_threshold_3D", + [([(129, 463)], 0), ([(1, 2011)], 0), ([(1, 2011)], 8)], +) +def test_triton_unified_attn_diffkv_fp8_query(seq_lens, seq_threshold_3D): + test_triton_unified_attn_diffkv_vs_reference( + seq_lens=seq_lens, + num_heads=(8, 2), + head_sizes=(192, 128), + sliding_window=None, + soft_cap=None, + dtype=torch.bfloat16, + kv_cache_dtype=current_platform.fp8_dtype(), + block_size=16, + seq_threshold_3D=seq_threshold_3D, + quant_query=True, + ) diff --git a/vllm/model_executor/models/mimo_v2.py b/vllm/model_executor/models/mimo_v2.py index 6267a37dc0..a6464c7c84 100644 --- a/vllm/model_executor/models/mimo_v2.py +++ b/vllm/model_executor/models/mimo_v2.py @@ -372,6 +372,10 @@ class MiMoV2FlashDecoderLayer(nn.Module): max_position_embeddings = getattr(config, "max_position_embeddings", 32768) v_scale = getattr(config, "attention_value_scale", None) + # cache_config.sliding_window holds MiMo's SWA window; global layers + # would otherwise fall back to it in Attention. + cache_config = vllm_config.cache_config + cache_config.sliding_window = None if self.is_compressed_softmax_layer(): self.self_attn = MiMoV2Attention( @@ -389,6 +393,7 @@ class MiMoV2FlashDecoderLayer(nn.Module): layer_id=layer_id, rope_theta=getattr(config, "swa_rope_theta", rope_theta), max_position_embeddings=max_position_embeddings, + cache_config=cache_config, quant_config=quant_config, partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0), prefix=f"{prefix}.self_attn", @@ -406,6 +411,7 @@ class MiMoV2FlashDecoderLayer(nn.Module): layer_id=layer_id, rope_theta=rope_theta, max_position_embeddings=max_position_embeddings, + cache_config=cache_config, quant_config=quant_config, partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0), prefix=f"{prefix}.self_attn", diff --git a/vllm/model_executor/models/mimo_v2_mtp.py b/vllm/model_executor/models/mimo_v2_mtp.py index c413b65ba9..fe6d64552d 100644 --- a/vllm/model_executor/models/mimo_v2_mtp.py +++ b/vllm/model_executor/models/mimo_v2_mtp.py @@ -24,7 +24,7 @@ import torch import torch.nn as nn from transformers import PreTrainedConfig -from vllm.config import VllmConfig +from vllm.config import CacheConfig, VllmConfig from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -64,6 +64,7 @@ class MiMoV2MTPLayer(nn.Module): self, config: PreTrainedConfig, prefix: str, + cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, ) -> None: super().__init__() @@ -100,6 +101,7 @@ class MiMoV2MTPLayer(nn.Module): layer_id=0, rope_theta=swa_rope_theta, max_position_embeddings=getattr(config, "max_position_embeddings", 32768), + cache_config=cache_config, quant_config=quant_config, partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0), prefix=f"{prefix}.self_attn", @@ -147,6 +149,7 @@ class _MiMoV2MTPLayers(nn.Module): self, config: PreTrainedConfig, num_mtp_layers: int, + cache_config: CacheConfig | None, quant_config: QuantizationConfig | None, prefix: str, ) -> None: @@ -156,6 +159,7 @@ class _MiMoV2MTPLayers(nn.Module): str(i): MiMoV2MTPLayer( config=config, prefix=f"{prefix}.{i}", + cache_config=cache_config, quant_config=quant_config, ) for i in range(num_mtp_layers) @@ -182,6 +186,7 @@ class MiMoV2MultiTokenPredictor(nn.Module): self.mtp = _MiMoV2MTPLayers( config=config, num_mtp_layers=num_mtp_layers, + cache_config=vllm_config.cache_config, quant_config=vllm_config.quant_config, prefix=maybe_prefix(prefix, "mtp.layers"), ) diff --git a/vllm/v1/attention/backends/triton_attn_diffkv.py b/vllm/v1/attention/backends/triton_attn_diffkv.py index 4b42838a15..932f483af3 100644 --- a/vllm/v1/attention/backends/triton_attn_diffkv.py +++ b/vllm/v1/attention/backends/triton_attn_diffkv.py @@ -70,10 +70,11 @@ class TritonAttentionDiffKVBackend(TritonAttentionBackend): # V head dim — set per layer via ``set_head_size_v`` before instantiation. head_size_v: int = 128 - # No FP8 / int8 KV cache for the DiffKV path yet; require fp16/bf16/fp32. supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ "auto", "bfloat16", + "fp8", + "fp8_e4m3", ] @classmethod @@ -110,16 +111,15 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - if is_quantized_kv_cache(self.kv_cache_dtype): + if ( + is_quantized_kv_cache(self.kv_cache_dtype) + and self.kv_cache_dtype + not in TritonAttentionDiffKVBackend.supported_kv_cache_dtypes + ): raise NotImplementedError( - "TritonAttentionDiffKVBackend does not yet support quantized " + "TritonAttentionDiffKVBackend supports only fp8/fp8_e4m3 quantized " f"KV cache (got kv_cache_dtype={self.kv_cache_dtype!r})." ) - if self._is_per_token_head_quant: - raise NotImplementedError( - "TritonAttentionDiffKVBackend does not support per-token-head " - "quantization." - ) if self.chunk_lookback > -1: raise NotImplementedError( "TritonAttentionDiffKVBackend does not support chunked " @@ -191,6 +191,9 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl): # Triton DiffKV kernels consume (B, N, H, D) cache views. kv_cache = kv_cache.transpose(1, 2) + quantized = is_quantized_kv_cache(self.kv_cache_dtype) + if quantized: + kv_cache = kv_cache.view(self.fp8_dtype) key_cache = kv_cache[..., :head_size_qk] value_cache = kv_cache[..., head_size_qk : head_size_qk + head_size_v] @@ -202,6 +205,11 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl): cu_seqlens_q=attn_metadata.query_start_loc, seqused_k=attn_metadata.seq_lens, softmax_scale=self.scale, + q_descale=( + layer._q_scale if quantized and query.dtype == self.fp8_dtype else None + ), + k_descale=layer._k_scale if quantized else None, + v_descale=layer._v_scale if quantized else None, causal=True, alibi_slopes=self.alibi_slopes, use_alibi_sqrt=self.use_alibi_sqrt, diff --git a/vllm/v1/attention/ops/triton_unified_attention_diffkv.py b/vllm/v1/attention/ops/triton_unified_attention_diffkv.py index eaf62b6bce..8480754256 100644 --- a/vllm/v1/attention/ops/triton_unified_attention_diffkv.py +++ b/vllm/v1/attention/ops/triton_unified_attention_diffkv.py @@ -60,6 +60,9 @@ def kernel_unified_attention_diffkv( query_ptr, key_cache_ptr, # view of packed cache: [..., :head_size_qk] value_cache_ptr, # view of packed cache: [..., head_size_qk:hqk+hv] + q_descale_ptr, + k_descale_ptr, + v_descale_ptr, sink_ptr, block_tables_ptr, seq_lens_ptr, @@ -83,6 +86,8 @@ def kernel_unified_attention_diffkv( USE_ALIBI_SQRT: tl.constexpr, USE_SOFTCAP: tl.constexpr, USE_SINKS: tl.constexpr, + USE_Q_SCALE: tl.constexpr, + USE_KV_SCALES: tl.constexpr, SLIDING_WINDOW: tl.constexpr, # Strides for both cache views (they share the same packed buffer, so # dims 0/1/2 strides match; only the per-head extent differs). @@ -154,6 +159,12 @@ def kernel_unified_attention_diffkv( other=0.0, ) + if USE_Q_SCALE: + scale *= tl.load(q_descale_ptr) + if USE_KV_SCALES: + scale *= tl.load(k_descale_ptr) + v_descale = tl.load(v_descale_ptr) + block_table_offset = seq_idx * block_table_stride M = init_softmax_M( @@ -262,6 +273,9 @@ def kernel_unified_attention_diffkv( acc += tl.dot(P.to(V.dtype), V) # ---- Epilogue -------------------------------------------------------- + if USE_KV_SCALES: + acc *= v_descale + if IS_3D: # Store per-segment partials; finalized by reduce_segments_diffkv. segm_output_offset = ( @@ -403,6 +417,9 @@ def unified_attention_diffkv( softmax_segm_output: torch.Tensor | None = None, softmax_segm_max: torch.Tensor | None = None, softmax_segm_expsum: torch.Tensor | None = None, + k_descale: torch.Tensor | None = None, + v_descale: torch.Tensor | None = None, + q_descale: torch.Tensor | None = None, ): assert causal, "Only causal attention is supported" @@ -470,6 +487,9 @@ def unified_attention_diffkv( query_ptr=q, key_cache_ptr=k, value_cache_ptr=v, + q_descale_ptr=q_descale, + k_descale_ptr=k_descale, + v_descale_ptr=v_descale, sink_ptr=sinks, block_tables_ptr=block_table, seq_lens_ptr=seqused_k, @@ -493,6 +513,8 @@ def unified_attention_diffkv( USE_ALIBI_SQRT=use_alibi_sqrt, USE_SOFTCAP=(softcap > 0), USE_SINKS=(sinks is not None), + USE_Q_SCALE=q_descale is not None, + USE_KV_SCALES=k_descale is not None, SLIDING_WINDOW=sliding_window_val, stride_k_cache_0=k.stride(0), stride_k_cache_1=k.stride(1),