[KV-Offloading][TP] : Expand replicated_layout detection to multi-group MLA (#57652)
This commit is contained in:
@@ -9,7 +9,7 @@ import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial
|
||||
from functools import partial, reduce
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, overload
|
||||
|
||||
from vllm import envs
|
||||
@@ -2409,6 +2409,26 @@ def get_kv_cache_groups(
|
||||
return groups
|
||||
|
||||
|
||||
def _layer_tp_replicas(spec: KVCacheSpec, tp_size: int, dcp_size: int) -> int:
|
||||
if not isinstance(spec, AttentionSpec) or spec.max_tp_shards is None:
|
||||
return 1
|
||||
if spec.dcp_sharded and dcp_size > 1:
|
||||
return 1
|
||||
return max(1, tp_size // spec.max_tp_shards)
|
||||
|
||||
|
||||
def kv_cache_groups_tp_replicas(
|
||||
groups: list[KVCacheGroupSpec], tp_size: int, dcp_size: int = 1
|
||||
) -> int:
|
||||
"""Consecutive TP ranks holding identical KV for every layer."""
|
||||
specs = [spec for g in groups for spec in iter_layer_specs(g.kv_cache_spec)]
|
||||
if not specs:
|
||||
return 1
|
||||
return reduce(
|
||||
math.gcd, (_layer_tp_replicas(s, tp_size, dcp_size) for s in specs), tp_size
|
||||
)
|
||||
|
||||
|
||||
def generate_scheduler_kv_cache_config(
|
||||
kv_cache_configs: list[KVCacheConfig],
|
||||
) -> KVCacheConfig:
|
||||
@@ -2823,6 +2843,13 @@ def get_kv_cache_configs(
|
||||
vllm_config, groups, min_num_blocks * _pool_bytes_per_block(groups)
|
||||
)
|
||||
|
||||
for kv_cache_config in kv_cache_configs:
|
||||
kv_cache_config.kv_tp_replicas = kv_cache_groups_tp_replicas(
|
||||
kv_cache_config.kv_cache_groups,
|
||||
vllm_config.parallel_config.tensor_parallel_size,
|
||||
vllm_config.parallel_config.decode_context_parallel_size,
|
||||
)
|
||||
|
||||
return kv_cache_configs
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user