[Bugfix][KV Offload] Scale UniformTypeKVCacheSpecs groups by DCP (#50883)
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -231,6 +231,26 @@ def _make_hybrid_kv_cache_config() -> KVCacheConfig:
|
||||
)
|
||||
|
||||
|
||||
def _mamba_spec() -> MambaSpec:
|
||||
return MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
|
||||
|
||||
def _uniform_spec(spec_kind: str) -> UniformTypeKVCacheSpecs:
|
||||
# DSA models merge their indexer and MLA layers into a UniformType group,
|
||||
# a container rather than an AttentionSpec, but still sharded across DCP.
|
||||
specs: dict[str, Any] = (
|
||||
{"mla_layer": _mla_spec(), "indexer_layer": _mla_spec(head_size=128)}
|
||||
if spec_kind == "attention"
|
||||
else {"mamba_layer0": _mamba_spec(), "mamba_layer1": _mamba_spec()}
|
||||
)
|
||||
return UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs=specs)
|
||||
|
||||
|
||||
def _make_mamba_hybrid_kv_cache_config() -> KVCacheConfig:
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
@@ -369,6 +389,61 @@ def test_dcp_scales_attention_but_not_mamba_group_blocks():
|
||||
] == [1, 3]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp_size,expected", [(1, 16), (2, 32)])
|
||||
def test_dcp_scales_uniform_type_attention_group_blocks(dcp_size, expected):
|
||||
config = _make_vllm_config(
|
||||
tensor_parallel_size=2, decode_context_parallel_size=dcp_size
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=64,
|
||||
layers=["mla_layer", "indexer_layer"],
|
||||
layer_stride=32,
|
||||
block_stride=8,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["mla_layer", "indexer_layer"], _uniform_spec("attention"))
|
||||
],
|
||||
)
|
||||
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
|
||||
assert offloading_config.groups[0].tokens_per_block == expected
|
||||
assert offloading_config.cache.tokens_per_hash == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"spec_kind,expected",
|
||||
[
|
||||
("attention", 32),
|
||||
# Mamba state is replicated across DCP ranks, so a uniform group of
|
||||
# Mamba layers keeps its span.
|
||||
("mamba", 16),
|
||||
],
|
||||
)
|
||||
def test_dcp_scales_uniform_type_group_alongside_mamba(spec_kind, expected):
|
||||
config = _make_vllm_config(tensor_parallel_size=2, decode_context_parallel_size=2)
|
||||
config.speculative_config = None
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer0", "layer1"], _uniform_spec(spec_kind)),
|
||||
KVCacheGroupSpec(["mamba_layer"], _mamba_spec()),
|
||||
],
|
||||
)
|
||||
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
|
||||
assert tuple(group.tokens_per_block for group in offloading_config.groups) == (
|
||||
expected,
|
||||
16,
|
||||
)
|
||||
|
||||
|
||||
def test_preserves_data_parallel_config():
|
||||
config = _make_vllm_config()
|
||||
config.parallel_config.data_parallel_index = 2
|
||||
|
||||
@@ -4,7 +4,10 @@
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
resolve_dcp_kv_block_size,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
FullAttentionSpec,
|
||||
@@ -41,13 +44,9 @@ def build_offloading_config(
|
||||
parallel_config = vllm_config.parallel_config
|
||||
groups = tuple(
|
||||
OffloadingGroupConfig(
|
||||
tokens_per_block=(
|
||||
group.kv_cache_spec.block_size
|
||||
* (
|
||||
parallel_config.decode_context_parallel_size
|
||||
if isinstance(group.kv_cache_spec, AttentionSpec)
|
||||
else 1
|
||||
)
|
||||
tokens_per_block=resolve_dcp_kv_block_size(
|
||||
group.kv_cache_spec,
|
||||
parallel_config.decode_context_parallel_size,
|
||||
),
|
||||
layer_names=tuple(group.layer_names),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user