[KV-Offloading][TP] : Expand replicated_layout detection to multi-group MLA (#57652)
This commit is contained in:
@@ -484,6 +484,9 @@ class HiSparseResidentSpec(KVCacheSpec):
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class AttentionSpec(KVCacheSpec):
|
||||
dcp_sharded: bool = True
|
||||
max_tp_shards: int | None = None
|
||||
"""Distinct shards this cache splits into across TP; TP ranks beyond this
|
||||
hold replicas. None: unknown, treated as not replicated."""
|
||||
num_kv_heads: int
|
||||
head_size: int
|
||||
dtype: torch.dtype
|
||||
@@ -618,6 +621,7 @@ class FullAttentionSpec(AttentionSpec):
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
max_tp_shards=specs[0].max_tp_shards,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -705,6 +709,7 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
max_tp_shards=specs[0].max_tp_shards,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -768,6 +773,7 @@ class RSWASpec(FullAttentionSpec):
|
||||
dtype=base.dtype,
|
||||
kv_quant_mode=base.kv_quant_mode,
|
||||
dcp_sharded=base.dcp_sharded,
|
||||
max_tp_shards=base.max_tp_shards,
|
||||
page_size_padded=base.page_size_padded,
|
||||
num_head_slots=base.num_head_slots,
|
||||
state_content_bytes=base.state_content_bytes,
|
||||
@@ -960,6 +966,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
bounded_replay_set = set(spec.bounded_replay for spec in specs)
|
||||
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
|
||||
assert len({spec.dcp_sharded for spec in specs}) == 1
|
||||
assert len({spec.max_tp_shards for spec in specs}) == 1
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
@@ -980,6 +987,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
max_tp_shards=specs[0].max_tp_shards,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -1197,6 +1205,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
max_tp_shards=specs[0].max_tp_shards,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -1483,6 +1492,9 @@ class KVCacheConfig:
|
||||
hisparse_shared_host_pool: bool = False
|
||||
"""Whether local TP ranks share one physical HiSparse host pool."""
|
||||
|
||||
kv_tp_replicas: int = 1
|
||||
"""Consecutive TP ranks holding identical KV for every layer (1: none)."""
|
||||
|
||||
@cached_property
|
||||
def transfer_group_ids(self) -> tuple[int, ...]:
|
||||
"""IDs of cache groups that participate in external KV transfer."""
|
||||
|
||||
Reference in New Issue
Block a user