[Attention][Model] FP8 KV cache for Triton DiffKV and MiMo-V2.6-Flash (#58128)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Luca Motz
2026-10-05 21:18:29 -07:00
committed by GitHub
co-authored by Codex Claude Opus 5.5 Lucas Wilkinson
parent 30ab428235
commit 21d93d0d8c
5 changed files with 111 additions and 15 deletions
+6
View File
@@ -372,6 +372,10 @@ class MiMoV2FlashDecoderLayer(nn.Module):
max_position_embeddings = getattr(config, "max_position_embeddings", 32768)
v_scale = getattr(config, "attention_value_scale", None)
# cache_config.sliding_window holds MiMo's SWA window; global layers
# would otherwise fall back to it in Attention.
cache_config = vllm_config.cache_config
cache_config.sliding_window = None
if self.is_compressed_softmax_layer():
self.self_attn = MiMoV2Attention(
@@ -389,6 +393,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
layer_id=layer_id,
rope_theta=getattr(config, "swa_rope_theta", rope_theta),
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
@@ -406,6 +411,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
layer_id=layer_id,
rope_theta=rope_theta,
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
+6 -1
View File
@@ -24,7 +24,7 @@ import torch
import torch.nn as nn
from transformers import PreTrainedConfig
from vllm.config import VllmConfig
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -64,6 +64,7 @@ class MiMoV2MTPLayer(nn.Module):
self,
config: PreTrainedConfig,
prefix: str,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
@@ -100,6 +101,7 @@ class MiMoV2MTPLayer(nn.Module):
layer_id=0,
rope_theta=swa_rope_theta,
max_position_embeddings=getattr(config, "max_position_embeddings", 32768),
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
@@ -147,6 +149,7 @@ class _MiMoV2MTPLayers(nn.Module):
self,
config: PreTrainedConfig,
num_mtp_layers: int,
cache_config: CacheConfig | None,
quant_config: QuantizationConfig | None,
prefix: str,
) -> None:
@@ -156,6 +159,7 @@ class _MiMoV2MTPLayers(nn.Module):
str(i): MiMoV2MTPLayer(
config=config,
prefix=f"{prefix}.{i}",
cache_config=cache_config,
quant_config=quant_config,
)
for i in range(num_mtp_layers)
@@ -182,6 +186,7 @@ class MiMoV2MultiTokenPredictor(nn.Module):
self.mtp = _MiMoV2MTPLayers(
config=config,
num_mtp_layers=num_mtp_layers,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "mtp.layers"),
)