[Attention][Model] FP8 KV cache for Triton DiffKV and MiMo-V2.6-Flash (#58128)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
Claude Opus 5.5
Lucas Wilkinson
parent
30ab428235
commit
21d93d0d8c
@@ -35,6 +35,8 @@ NUM_HEADS = [(4, 4), (8, 2), (5, 1)]
|
||||
HEAD_SIZES = [(128, 128), (192, 128)]
|
||||
BLOCK_SIZES = [16]
|
||||
DTYPES = [torch.bfloat16]
|
||||
# None keeps the KV cache in the query dtype.
|
||||
KV_CACHE_DTYPES = [None, current_platform.fp8_dtype()]
|
||||
|
||||
NUM_BLOCKS = 2048
|
||||
|
||||
@@ -80,6 +82,7 @@ def _alloc_segm_buffers(seq_threshold_3D: int, num_query_heads: int, head_size_v
|
||||
@pytest.mark.parametrize("sliding_window", [None, 128])
|
||||
@pytest.mark.parametrize("soft_cap", [None, 50.0])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("seq_threshold_3D", SEQ_THRESHOLD_3D_VALUES)
|
||||
@torch.inference_mode()
|
||||
def test_triton_unified_attn_diffkv_vs_reference(
|
||||
@@ -89,13 +92,17 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
sliding_window: int | None,
|
||||
soft_cap: float | None,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: torch.dtype | None,
|
||||
block_size: int,
|
||||
seq_threshold_3D: int,
|
||||
quant_query: bool = False,
|
||||
) -> None:
|
||||
head_size_qk, head_size_v = head_sizes
|
||||
|
||||
# Keep the FA3/FA4 comparison on NVIDIA; ROCm uses the PyTorch oracle.
|
||||
if not current_platform.is_rocm():
|
||||
# FA3/FA4 is the reference on NVIDIA; ROCm and the FP8 KV cache (FA would
|
||||
# also quantize Q) use the PyTorch oracle.
|
||||
use_torch_ref = current_platform.is_rocm() or kv_cache_dtype is not None
|
||||
if not use_torch_ref:
|
||||
fa_version = get_flash_attn_version(
|
||||
head_size=head_size_qk, head_size_v=head_size_v
|
||||
)
|
||||
@@ -126,6 +133,20 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
)
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk:]
|
||||
q_descale = k_descale = v_descale = None
|
||||
if kv_cache_dtype is not None:
|
||||
# Non-power-of-two scales exercise descale rounding.
|
||||
k_descale = torch.tensor(0.3, dtype=torch.float32)
|
||||
v_descale = torch.tensor(0.7, dtype=torch.float32)
|
||||
kv_cache = torch.cat(
|
||||
[key_cache / k_descale, value_cache / v_descale], dim=-1
|
||||
).to(kv_cache_dtype)
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk:]
|
||||
|
||||
if quant_query:
|
||||
q_descale = torch.tensor(0.2, dtype=torch.float32)
|
||||
query = (query / q_descale).to(kv_cache_dtype)
|
||||
|
||||
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
|
||||
dim=0, dtype=torch.int32
|
||||
@@ -137,12 +158,20 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
if use_torch_ref:
|
||||
ref_key_cache = key_cache.float()
|
||||
ref_value_cache = value_cache.float()
|
||||
if kv_cache_dtype is not None:
|
||||
ref_key_cache *= k_descale
|
||||
ref_value_cache *= v_descale
|
||||
# FP32 also keeps the helper's in-place scaling off the kernel input.
|
||||
ref_query = query.float()
|
||||
if quant_query:
|
||||
ref_query *= q_descale
|
||||
ref_out = ref_paged_attn(
|
||||
query.float(),
|
||||
key_cache.float(),
|
||||
value_cache.float(),
|
||||
ref_query,
|
||||
ref_key_cache,
|
||||
ref_value_cache,
|
||||
query_lens,
|
||||
kv_lens,
|
||||
block_tables,
|
||||
@@ -200,6 +229,32 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
softmax_segm_output=segm_output,
|
||||
softmax_segm_max=segm_max,
|
||||
softmax_segm_expsum=segm_expsum,
|
||||
q_descale=q_descale,
|
||||
k_descale=k_descale,
|
||||
v_descale=v_descale,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(triton_out, ref_out, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not current_platform.has_device_capability(89),
|
||||
reason="FP8 query requires CUDA SM89+",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,seq_threshold_3D",
|
||||
[([(129, 463)], 0), ([(1, 2011)], 0), ([(1, 2011)], 8)],
|
||||
)
|
||||
def test_triton_unified_attn_diffkv_fp8_query(seq_lens, seq_threshold_3D):
|
||||
test_triton_unified_attn_diffkv_vs_reference(
|
||||
seq_lens=seq_lens,
|
||||
num_heads=(8, 2),
|
||||
head_sizes=(192, 128),
|
||||
sliding_window=None,
|
||||
soft_cap=None,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype=current_platform.fp8_dtype(),
|
||||
block_size=16,
|
||||
seq_threshold_3D=seq_threshold_3D,
|
||||
quant_query=True,
|
||||
)
|
||||
|
||||
@@ -372,6 +372,10 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 32768)
|
||||
|
||||
v_scale = getattr(config, "attention_value_scale", None)
|
||||
# cache_config.sliding_window holds MiMo's SWA window; global layers
|
||||
# would otherwise fall back to it in Attention.
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_config.sliding_window = None
|
||||
|
||||
if self.is_compressed_softmax_layer():
|
||||
self.self_attn = MiMoV2Attention(
|
||||
@@ -389,6 +393,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
rope_theta=getattr(config, "swa_rope_theta", rope_theta),
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
||||
prefix=f"{prefix}.self_attn",
|
||||
@@ -406,6 +411,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
rope_theta=rope_theta,
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
||||
prefix=f"{prefix}.self_attn",
|
||||
|
||||
@@ -24,7 +24,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -64,6 +64,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
self,
|
||||
config: PreTrainedConfig,
|
||||
prefix: str,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -100,6 +101,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
layer_id=0,
|
||||
rope_theta=swa_rope_theta,
|
||||
max_position_embeddings=getattr(config, "max_position_embeddings", 32768),
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
||||
prefix=f"{prefix}.self_attn",
|
||||
@@ -147,6 +149,7 @@ class _MiMoV2MTPLayers(nn.Module):
|
||||
self,
|
||||
config: PreTrainedConfig,
|
||||
num_mtp_layers: int,
|
||||
cache_config: CacheConfig | None,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
@@ -156,6 +159,7 @@ class _MiMoV2MTPLayers(nn.Module):
|
||||
str(i): MiMoV2MTPLayer(
|
||||
config=config,
|
||||
prefix=f"{prefix}.{i}",
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(num_mtp_layers)
|
||||
@@ -182,6 +186,7 @@ class MiMoV2MultiTokenPredictor(nn.Module):
|
||||
self.mtp = _MiMoV2MTPLayers(
|
||||
config=config,
|
||||
num_mtp_layers=num_mtp_layers,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "mtp.layers"),
|
||||
)
|
||||
|
||||
@@ -70,10 +70,11 @@ class TritonAttentionDiffKVBackend(TritonAttentionBackend):
|
||||
# V head dim — set per layer via ``set_head_size_v`` before instantiation.
|
||||
head_size_v: int = 128
|
||||
|
||||
# No FP8 / int8 KV cache for the DiffKV path yet; require fp16/bf16/fp32.
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
@@ -110,16 +111,15 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
if (
|
||||
is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and self.kv_cache_dtype
|
||||
not in TritonAttentionDiffKVBackend.supported_kv_cache_dtypes
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"TritonAttentionDiffKVBackend does not yet support quantized "
|
||||
"TritonAttentionDiffKVBackend supports only fp8/fp8_e4m3 quantized "
|
||||
f"KV cache (got kv_cache_dtype={self.kv_cache_dtype!r})."
|
||||
)
|
||||
if self._is_per_token_head_quant:
|
||||
raise NotImplementedError(
|
||||
"TritonAttentionDiffKVBackend does not support per-token-head "
|
||||
"quantization."
|
||||
)
|
||||
if self.chunk_lookback > -1:
|
||||
raise NotImplementedError(
|
||||
"TritonAttentionDiffKVBackend does not support chunked "
|
||||
@@ -191,6 +191,9 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
|
||||
|
||||
# Triton DiffKV kernels consume (B, N, H, D) cache views.
|
||||
kv_cache = kv_cache.transpose(1, 2)
|
||||
quantized = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
if quantized:
|
||||
kv_cache = kv_cache.view(self.fp8_dtype)
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk : head_size_qk + head_size_v]
|
||||
|
||||
@@ -202,6 +205,11 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
|
||||
cu_seqlens_q=attn_metadata.query_start_loc,
|
||||
seqused_k=attn_metadata.seq_lens,
|
||||
softmax_scale=self.scale,
|
||||
q_descale=(
|
||||
layer._q_scale if quantized and query.dtype == self.fp8_dtype else None
|
||||
),
|
||||
k_descale=layer._k_scale if quantized else None,
|
||||
v_descale=layer._v_scale if quantized else None,
|
||||
causal=True,
|
||||
alibi_slopes=self.alibi_slopes,
|
||||
use_alibi_sqrt=self.use_alibi_sqrt,
|
||||
|
||||
@@ -60,6 +60,9 @@ def kernel_unified_attention_diffkv(
|
||||
query_ptr,
|
||||
key_cache_ptr, # view of packed cache: [..., :head_size_qk]
|
||||
value_cache_ptr, # view of packed cache: [..., head_size_qk:hqk+hv]
|
||||
q_descale_ptr,
|
||||
k_descale_ptr,
|
||||
v_descale_ptr,
|
||||
sink_ptr,
|
||||
block_tables_ptr,
|
||||
seq_lens_ptr,
|
||||
@@ -83,6 +86,8 @@ def kernel_unified_attention_diffkv(
|
||||
USE_ALIBI_SQRT: tl.constexpr,
|
||||
USE_SOFTCAP: tl.constexpr,
|
||||
USE_SINKS: tl.constexpr,
|
||||
USE_Q_SCALE: tl.constexpr,
|
||||
USE_KV_SCALES: tl.constexpr,
|
||||
SLIDING_WINDOW: tl.constexpr,
|
||||
# Strides for both cache views (they share the same packed buffer, so
|
||||
# dims 0/1/2 strides match; only the per-head extent differs).
|
||||
@@ -154,6 +159,12 @@ def kernel_unified_attention_diffkv(
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
if USE_Q_SCALE:
|
||||
scale *= tl.load(q_descale_ptr)
|
||||
if USE_KV_SCALES:
|
||||
scale *= tl.load(k_descale_ptr)
|
||||
v_descale = tl.load(v_descale_ptr)
|
||||
|
||||
block_table_offset = seq_idx * block_table_stride
|
||||
|
||||
M = init_softmax_M(
|
||||
@@ -262,6 +273,9 @@ def kernel_unified_attention_diffkv(
|
||||
acc += tl.dot(P.to(V.dtype), V)
|
||||
|
||||
# ---- Epilogue --------------------------------------------------------
|
||||
if USE_KV_SCALES:
|
||||
acc *= v_descale
|
||||
|
||||
if IS_3D:
|
||||
# Store per-segment partials; finalized by reduce_segments_diffkv.
|
||||
segm_output_offset = (
|
||||
@@ -403,6 +417,9 @@ def unified_attention_diffkv(
|
||||
softmax_segm_output: torch.Tensor | None = None,
|
||||
softmax_segm_max: torch.Tensor | None = None,
|
||||
softmax_segm_expsum: torch.Tensor | None = None,
|
||||
k_descale: torch.Tensor | None = None,
|
||||
v_descale: torch.Tensor | None = None,
|
||||
q_descale: torch.Tensor | None = None,
|
||||
):
|
||||
assert causal, "Only causal attention is supported"
|
||||
|
||||
@@ -470,6 +487,9 @@ def unified_attention_diffkv(
|
||||
query_ptr=q,
|
||||
key_cache_ptr=k,
|
||||
value_cache_ptr=v,
|
||||
q_descale_ptr=q_descale,
|
||||
k_descale_ptr=k_descale,
|
||||
v_descale_ptr=v_descale,
|
||||
sink_ptr=sinks,
|
||||
block_tables_ptr=block_table,
|
||||
seq_lens_ptr=seqused_k,
|
||||
@@ -493,6 +513,8 @@ def unified_attention_diffkv(
|
||||
USE_ALIBI_SQRT=use_alibi_sqrt,
|
||||
USE_SOFTCAP=(softcap > 0),
|
||||
USE_SINKS=(sinks is not None),
|
||||
USE_Q_SCALE=q_descale is not None,
|
||||
USE_KV_SCALES=k_descale is not None,
|
||||
SLIDING_WINDOW=sliding_window_val,
|
||||
stride_k_cache_0=k.stride(0),
|
||||
stride_k_cache_1=k.stride(1),
|
||||
|
||||
Reference in New Issue
Block a user