[KV-Offloading][TP] : Expand replicated_layout detection to multi-group MLA (#57652)

This commit is contained in:
Varun Sundar Rabindranath
2026-09-30 09:11:03 -04:00
committed by GitHub
parent f28a508162
commit 5463fe4962
10 changed files with 553 additions and 91 deletions
@@ -157,15 +157,10 @@ def build_offloading_config(
)
replicated_layout = (
vllm_config.model_config.use_mla
# Exact type: fail closed on wrappers and sliding-window variants.
and type(single_group_spec) is MLAAttentionSpec
# Page accounting: one MLA page per layer, no packed/mixed rows.
and worker_kv_bytes_per_block > 0
and worker_kv_bytes_per_block
== single_group_spec.page_size_bytes
* len(kv_cache_config.kv_cache_groups[0].layer_names)
# Safe MVP boundary: TP-only, no other parallel axes.
and parallel_config.tensor_parallel_size > 1
and kv_cache_config.kv_tp_replicas == parallel_config.tensor_parallel_size
and worker_kv_bytes_per_block > 0
# Safe MVP boundary: TP-only, no other parallel axes.
and parallel_config.pipeline_parallel_size == 1
and parallel_config.prefill_context_parallel_size == 1
and parallel_config.decode_context_parallel_size == 1
@@ -233,9 +228,10 @@ def build_offloading_config(
and parallel_config.world_size == tp_size
)
if canonical_layout and is_parallelism_agnostic:
if canonical_layout:
replicated_layout = (
all(
is_parallelism_agnostic
and all(
type(spec) is MLAAttentionSpec
for _, group in selected_groups
for spec in iter_layer_specs(group.kv_cache_spec)
@@ -1350,6 +1350,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
common_kwargs = dict(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_size,
dtype=kv_cache_dtype,
cache_dtype_str=self.kv_cache_dtype,
@@ -658,6 +658,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_dim,
dtype=self.dtype,
cache_role=SparseCacheRole.INDEXER,
+2
View File
@@ -865,6 +865,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
return MLAAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_dim,
dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype,
tokens_per_state=self.compress_ratio,
@@ -910,6 +911,7 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_dim,
dtype=self.dtype,
tokens_per_state=self.compress_ratio,
+1
View File
@@ -184,6 +184,7 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
return SlidingWindowMLASpec( # only has one vector instead of K + V
block_size=self.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.state_dim,
dtype=self.dtype,
sliding_window=self.sliding_window,
@@ -124,6 +124,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
bounded_replay=self.bounded_replay,
block_size=self.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_dim,
dtype=self.dtype,
sliding_window=self.window_size,
+28 -1
View File
@@ -9,7 +9,7 @@ import os
from collections import defaultdict
from collections.abc import Callable, Iterable, Iterator, Sequence
from dataclasses import dataclass, field, replace
from functools import partial
from functools import partial, reduce
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, overload
from vllm import envs
@@ -2409,6 +2409,26 @@ def get_kv_cache_groups(
return groups
def _layer_tp_replicas(spec: KVCacheSpec, tp_size: int, dcp_size: int) -> int:
if not isinstance(spec, AttentionSpec) or spec.max_tp_shards is None:
return 1
if spec.dcp_sharded and dcp_size > 1:
return 1
return max(1, tp_size // spec.max_tp_shards)
def kv_cache_groups_tp_replicas(
groups: list[KVCacheGroupSpec], tp_size: int, dcp_size: int = 1
) -> int:
"""Consecutive TP ranks holding identical KV for every layer."""
specs = [spec for g in groups for spec in iter_layer_specs(g.kv_cache_spec)]
if not specs:
return 1
return reduce(
math.gcd, (_layer_tp_replicas(s, tp_size, dcp_size) for s in specs), tp_size
)
def generate_scheduler_kv_cache_config(
kv_cache_configs: list[KVCacheConfig],
) -> KVCacheConfig:
@@ -2823,6 +2843,13 @@ def get_kv_cache_configs(
vllm_config, groups, min_num_blocks * _pool_bytes_per_block(groups)
)
for kv_cache_config in kv_cache_configs:
kv_cache_config.kv_tp_replicas = kv_cache_groups_tp_replicas(
kv_cache_config.kv_cache_groups,
vllm_config.parallel_config.tensor_parallel_size,
vllm_config.parallel_config.decode_context_parallel_size,
)
return kv_cache_configs
+12
View File
@@ -484,6 +484,9 @@ class HiSparseResidentSpec(KVCacheSpec):
@dataclass(frozen=True, kw_only=True)
class AttentionSpec(KVCacheSpec):
dcp_sharded: bool = True
max_tp_shards: int | None = None
"""Distinct shards this cache splits into across TP; TP ranks beyond this
hold replicas. None: unknown, treated as not replicated."""
num_kv_heads: int
head_size: int
dtype: torch.dtype
@@ -618,6 +621,7 @@ class FullAttentionSpec(AttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
dcp_sharded=specs[0].dcp_sharded,
max_tp_shards=specs[0].max_tp_shards,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -705,6 +709,7 @@ class MLAAttentionSpec(FullAttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
dcp_sharded=specs[0].dcp_sharded,
max_tp_shards=specs[0].max_tp_shards,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -768,6 +773,7 @@ class RSWASpec(FullAttentionSpec):
dtype=base.dtype,
kv_quant_mode=base.kv_quant_mode,
dcp_sharded=base.dcp_sharded,
max_tp_shards=base.max_tp_shards,
page_size_padded=base.page_size_padded,
num_head_slots=base.num_head_slots,
state_content_bytes=base.state_content_bytes,
@@ -960,6 +966,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
bounded_replay_set = set(spec.bounded_replay for spec in specs)
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
assert len({spec.dcp_sharded for spec in specs}) == 1
assert len({spec.max_tp_shards for spec in specs}) == 1
assert (
len(cache_dtype_str_set) == 1
and len(tokens_per_state_set) == 1
@@ -980,6 +987,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
head_size=specs[0].head_size,
dtype=specs[0].dtype,
dcp_sharded=specs[0].dcp_sharded,
max_tp_shards=specs[0].max_tp_shards,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -1197,6 +1205,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
dcp_sharded=specs[0].dcp_sharded,
max_tp_shards=specs[0].max_tp_shards,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -1483,6 +1492,9 @@ class KVCacheConfig:
hisparse_shared_host_pool: bool = False
"""Whether local TP ranks share one physical HiSparse host pool."""
kv_tp_replicas: int = 1
"""Consecutive TP ranks holding identical KV for every layer (1: none)."""
@cached_property
def transfer_group_ids(self) -> tuple[int, ...]:
"""IDs of cache groups that participate in external KV transfer."""