[Attention][Model] FP8 KV cache for Triton DiffKV and MiMo-V2.6-Flash (#58128)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Luca Motz
2026-10-05 21:18:29 -07:00
committed by GitHub
co-authored by Codex Claude Opus 5.5 Lucas Wilkinson
parent 30ab428235
commit 21d93d0d8c
5 changed files with 111 additions and 15 deletions
+6
View File
@@ -372,6 +372,10 @@ class MiMoV2FlashDecoderLayer(nn.Module):
max_position_embeddings = getattr(config, "max_position_embeddings", 32768)
v_scale = getattr(config, "attention_value_scale", None)
# cache_config.sliding_window holds MiMo's SWA window; global layers
# would otherwise fall back to it in Attention.
cache_config = vllm_config.cache_config
cache_config.sliding_window = None
if self.is_compressed_softmax_layer():
self.self_attn = MiMoV2Attention(
@@ -389,6 +393,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
layer_id=layer_id,
rope_theta=getattr(config, "swa_rope_theta", rope_theta),
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
@@ -406,6 +411,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
layer_id=layer_id,
rope_theta=rope_theta,
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",