[KV-Offloading][TP] : Expand replicated_layout detection to multi-group MLA (#57652)

This commit is contained in:
Varun Sundar Rabindranath
2026-09-30 09:11:03 -04:00
committed by GitHub
parent f28a508162
commit 5463fe4962
10 changed files with 553 additions and 91 deletions
+2
View File
@@ -865,6 +865,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
return MLAAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_dim,
dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype,
tokens_per_state=self.compress_ratio,
@@ -910,6 +911,7 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_dim,
dtype=self.dtype,
tokens_per_state=self.compress_ratio,
+1
View File
@@ -184,6 +184,7 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
return SlidingWindowMLASpec( # only has one vector instead of K + V
block_size=self.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.state_dim,
dtype=self.dtype,
sliding_window=self.sliding_window,