[KV-Offloading][TP] : Expand replicated_layout detection to multi-group MLA (#57652)

This commit is contained in:
Varun Sundar Rabindranath
2026-09-30 09:11:03 -04:00
committed by GitHub
parent f28a508162
commit 5463fe4962
10 changed files with 553 additions and 91 deletions
+28 -1
View File
@@ -9,7 +9,7 @@ import os
from collections import defaultdict
from collections.abc import Callable, Iterable, Iterator, Sequence
from dataclasses import dataclass, field, replace
from functools import partial
from functools import partial, reduce
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, overload
from vllm import envs
@@ -2409,6 +2409,26 @@ def get_kv_cache_groups(
return groups
def _layer_tp_replicas(spec: KVCacheSpec, tp_size: int, dcp_size: int) -> int:
if not isinstance(spec, AttentionSpec) or spec.max_tp_shards is None:
return 1
if spec.dcp_sharded and dcp_size > 1:
return 1
return max(1, tp_size // spec.max_tp_shards)
def kv_cache_groups_tp_replicas(
groups: list[KVCacheGroupSpec], tp_size: int, dcp_size: int = 1
) -> int:
"""Consecutive TP ranks holding identical KV for every layer."""
specs = [spec for g in groups for spec in iter_layer_specs(g.kv_cache_spec)]
if not specs:
return 1
return reduce(
math.gcd, (_layer_tp_replicas(s, tp_size, dcp_size) for s in specs), tp_size
)
def generate_scheduler_kv_cache_config(
kv_cache_configs: list[KVCacheConfig],
) -> KVCacheConfig:
@@ -2823,6 +2843,13 @@ def get_kv_cache_configs(
vllm_config, groups, min_num_blocks * _pool_bytes_per_block(groups)
)
for kv_cache_config in kv_cache_configs:
kv_cache_config.kv_tp_replicas = kv_cache_groups_tp_replicas(
kv_cache_config.kv_cache_groups,
vllm_config.parallel_config.tensor_parallel_size,
vllm_config.parallel_config.decode_context_parallel_size,
)
return kv_cache_configs