[KV-Offloading][TP] : Expand replicated_layout detection to multi-group MLA (#57652)

This commit is contained in:
Varun Sundar Rabindranath
2026-09-30 09:11:03 -04:00
committed by GitHub
parent f28a508162
commit 5463fe4962
10 changed files with 553 additions and 91 deletions
+12
View File
@@ -484,6 +484,9 @@ class HiSparseResidentSpec(KVCacheSpec):
@dataclass(frozen=True, kw_only=True)
class AttentionSpec(KVCacheSpec):
dcp_sharded: bool = True
max_tp_shards: int | None = None
"""Distinct shards this cache splits into across TP; TP ranks beyond this
hold replicas. None: unknown, treated as not replicated."""
num_kv_heads: int
head_size: int
dtype: torch.dtype
@@ -618,6 +621,7 @@ class FullAttentionSpec(AttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
dcp_sharded=specs[0].dcp_sharded,
max_tp_shards=specs[0].max_tp_shards,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -705,6 +709,7 @@ class MLAAttentionSpec(FullAttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
dcp_sharded=specs[0].dcp_sharded,
max_tp_shards=specs[0].max_tp_shards,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -768,6 +773,7 @@ class RSWASpec(FullAttentionSpec):
dtype=base.dtype,
kv_quant_mode=base.kv_quant_mode,
dcp_sharded=base.dcp_sharded,
max_tp_shards=base.max_tp_shards,
page_size_padded=base.page_size_padded,
num_head_slots=base.num_head_slots,
state_content_bytes=base.state_content_bytes,
@@ -960,6 +966,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
bounded_replay_set = set(spec.bounded_replay for spec in specs)
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
assert len({spec.dcp_sharded for spec in specs}) == 1
assert len({spec.max_tp_shards for spec in specs}) == 1
assert (
len(cache_dtype_str_set) == 1
and len(tokens_per_state_set) == 1
@@ -980,6 +987,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
head_size=specs[0].head_size,
dtype=specs[0].dtype,
dcp_sharded=specs[0].dcp_sharded,
max_tp_shards=specs[0].max_tp_shards,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -1197,6 +1205,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
dcp_sharded=specs[0].dcp_sharded,
max_tp_shards=specs[0].max_tp_shards,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -1483,6 +1492,9 @@ class KVCacheConfig:
hisparse_shared_host_pool: bool = False
"""Whether local TP ranks share one physical HiSparse host pool."""
kv_tp_replicas: int = 1
"""Consecutive TP ranks holding identical KV for every layer (1: none)."""
@cached_property
def transfer_group_ids(self) -> tuple[int, ...]:
"""IDs of cache groups that participate in external KV transfer."""