[KV-Offloading][TP] : Expand replicated_layout detection to multi-group MLA (#57652)
This commit is contained in:
@@ -157,15 +157,10 @@ def build_offloading_config(
|
||||
)
|
||||
replicated_layout = (
|
||||
vllm_config.model_config.use_mla
|
||||
# Exact type: fail closed on wrappers and sliding-window variants.
|
||||
and type(single_group_spec) is MLAAttentionSpec
|
||||
# Page accounting: one MLA page per layer, no packed/mixed rows.
|
||||
and worker_kv_bytes_per_block > 0
|
||||
and worker_kv_bytes_per_block
|
||||
== single_group_spec.page_size_bytes
|
||||
* len(kv_cache_config.kv_cache_groups[0].layer_names)
|
||||
# Safe MVP boundary: TP-only, no other parallel axes.
|
||||
and parallel_config.tensor_parallel_size > 1
|
||||
and kv_cache_config.kv_tp_replicas == parallel_config.tensor_parallel_size
|
||||
and worker_kv_bytes_per_block > 0
|
||||
# Safe MVP boundary: TP-only, no other parallel axes.
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
and parallel_config.prefill_context_parallel_size == 1
|
||||
and parallel_config.decode_context_parallel_size == 1
|
||||
@@ -233,9 +228,10 @@ def build_offloading_config(
|
||||
and parallel_config.world_size == tp_size
|
||||
)
|
||||
|
||||
if canonical_layout and is_parallelism_agnostic:
|
||||
if canonical_layout:
|
||||
replicated_layout = (
|
||||
all(
|
||||
is_parallelism_agnostic
|
||||
and all(
|
||||
type(spec) is MLAAttentionSpec
|
||||
for _, group in selected_groups
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
|
||||
@@ -1350,6 +1350,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
common_kwargs = dict(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=self.head_size,
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
|
||||
@@ -658,6 +658,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
return MLAAttentionSpec(
|
||||
block_size=self.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.dtype,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
|
||||
@@ -865,6 +865,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
return MLAAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype,
|
||||
tokens_per_state=self.compress_ratio,
|
||||
@@ -910,6 +911,7 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
return MLAAttentionSpec(
|
||||
block_size=self.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.dtype,
|
||||
tokens_per_state=self.compress_ratio,
|
||||
|
||||
@@ -184,6 +184,7 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
return SlidingWindowMLASpec( # only has one vector instead of K + V
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=self.state_dim,
|
||||
dtype=self.dtype,
|
||||
sliding_window=self.sliding_window,
|
||||
|
||||
@@ -124,6 +124,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
bounded_replay=self.bounded_replay,
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.dtype,
|
||||
sliding_window=self.window_size,
|
||||
|
||||
@@ -9,7 +9,7 @@ import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial
|
||||
from functools import partial, reduce
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, overload
|
||||
|
||||
from vllm import envs
|
||||
@@ -2409,6 +2409,26 @@ def get_kv_cache_groups(
|
||||
return groups
|
||||
|
||||
|
||||
def _layer_tp_replicas(spec: KVCacheSpec, tp_size: int, dcp_size: int) -> int:
|
||||
if not isinstance(spec, AttentionSpec) or spec.max_tp_shards is None:
|
||||
return 1
|
||||
if spec.dcp_sharded and dcp_size > 1:
|
||||
return 1
|
||||
return max(1, tp_size // spec.max_tp_shards)
|
||||
|
||||
|
||||
def kv_cache_groups_tp_replicas(
|
||||
groups: list[KVCacheGroupSpec], tp_size: int, dcp_size: int = 1
|
||||
) -> int:
|
||||
"""Consecutive TP ranks holding identical KV for every layer."""
|
||||
specs = [spec for g in groups for spec in iter_layer_specs(g.kv_cache_spec)]
|
||||
if not specs:
|
||||
return 1
|
||||
return reduce(
|
||||
math.gcd, (_layer_tp_replicas(s, tp_size, dcp_size) for s in specs), tp_size
|
||||
)
|
||||
|
||||
|
||||
def generate_scheduler_kv_cache_config(
|
||||
kv_cache_configs: list[KVCacheConfig],
|
||||
) -> KVCacheConfig:
|
||||
@@ -2823,6 +2843,13 @@ def get_kv_cache_configs(
|
||||
vllm_config, groups, min_num_blocks * _pool_bytes_per_block(groups)
|
||||
)
|
||||
|
||||
for kv_cache_config in kv_cache_configs:
|
||||
kv_cache_config.kv_tp_replicas = kv_cache_groups_tp_replicas(
|
||||
kv_cache_config.kv_cache_groups,
|
||||
vllm_config.parallel_config.tensor_parallel_size,
|
||||
vllm_config.parallel_config.decode_context_parallel_size,
|
||||
)
|
||||
|
||||
return kv_cache_configs
|
||||
|
||||
|
||||
|
||||
@@ -484,6 +484,9 @@ class HiSparseResidentSpec(KVCacheSpec):
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class AttentionSpec(KVCacheSpec):
|
||||
dcp_sharded: bool = True
|
||||
max_tp_shards: int | None = None
|
||||
"""Distinct shards this cache splits into across TP; TP ranks beyond this
|
||||
hold replicas. None: unknown, treated as not replicated."""
|
||||
num_kv_heads: int
|
||||
head_size: int
|
||||
dtype: torch.dtype
|
||||
@@ -618,6 +621,7 @@ class FullAttentionSpec(AttentionSpec):
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
max_tp_shards=specs[0].max_tp_shards,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -705,6 +709,7 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
max_tp_shards=specs[0].max_tp_shards,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -768,6 +773,7 @@ class RSWASpec(FullAttentionSpec):
|
||||
dtype=base.dtype,
|
||||
kv_quant_mode=base.kv_quant_mode,
|
||||
dcp_sharded=base.dcp_sharded,
|
||||
max_tp_shards=base.max_tp_shards,
|
||||
page_size_padded=base.page_size_padded,
|
||||
num_head_slots=base.num_head_slots,
|
||||
state_content_bytes=base.state_content_bytes,
|
||||
@@ -960,6 +966,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
bounded_replay_set = set(spec.bounded_replay for spec in specs)
|
||||
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
|
||||
assert len({spec.dcp_sharded for spec in specs}) == 1
|
||||
assert len({spec.max_tp_shards for spec in specs}) == 1
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
@@ -980,6 +987,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
max_tp_shards=specs[0].max_tp_shards,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -1197,6 +1205,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
max_tp_shards=specs[0].max_tp_shards,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -1483,6 +1492,9 @@ class KVCacheConfig:
|
||||
hisparse_shared_host_pool: bool = False
|
||||
"""Whether local TP ranks share one physical HiSparse host pool."""
|
||||
|
||||
kv_tp_replicas: int = 1
|
||||
"""Consecutive TP ranks holding identical KV for every layer (1: none)."""
|
||||
|
||||
@cached_property
|
||||
def transfer_group_ids(self) -> tuple[int, ...]:
|
||||
"""IDs of cache groups that participate in external KV transfer."""
|
||||
|
||||
Reference in New Issue
Block a user