[Attention][Model] FP8 KV cache for Triton DiffKV and MiMo-V2.6-Flash (#58128)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
Claude Opus 5.5
Lucas Wilkinson
parent
30ab428235
commit
21d93d0d8c
@@ -372,6 +372,10 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 32768)
|
||||
|
||||
v_scale = getattr(config, "attention_value_scale", None)
|
||||
# cache_config.sliding_window holds MiMo's SWA window; global layers
|
||||
# would otherwise fall back to it in Attention.
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_config.sliding_window = None
|
||||
|
||||
if self.is_compressed_softmax_layer():
|
||||
self.self_attn = MiMoV2Attention(
|
||||
@@ -389,6 +393,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
rope_theta=getattr(config, "swa_rope_theta", rope_theta),
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
||||
prefix=f"{prefix}.self_attn",
|
||||
@@ -406,6 +411,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
rope_theta=rope_theta,
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
||||
prefix=f"{prefix}.self_attn",
|
||||
|
||||
@@ -24,7 +24,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -64,6 +64,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
self,
|
||||
config: PreTrainedConfig,
|
||||
prefix: str,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -100,6 +101,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
layer_id=0,
|
||||
rope_theta=swa_rope_theta,
|
||||
max_position_embeddings=getattr(config, "max_position_embeddings", 32768),
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
||||
prefix=f"{prefix}.self_attn",
|
||||
@@ -147,6 +149,7 @@ class _MiMoV2MTPLayers(nn.Module):
|
||||
self,
|
||||
config: PreTrainedConfig,
|
||||
num_mtp_layers: int,
|
||||
cache_config: CacheConfig | None,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
@@ -156,6 +159,7 @@ class _MiMoV2MTPLayers(nn.Module):
|
||||
str(i): MiMoV2MTPLayer(
|
||||
config=config,
|
||||
prefix=f"{prefix}.{i}",
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(num_mtp_layers)
|
||||
@@ -182,6 +186,7 @@ class MiMoV2MultiTokenPredictor(nn.Module):
|
||||
self.mtp = _MiMoV2MTPLayers(
|
||||
config=config,
|
||||
num_mtp_layers=num_mtp_layers,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "mtp.layers"),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user