diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_config.py b/tests/v1/kv_connector/unit/offloading_connector/test_config.py index 5e2ef5acf1..33e6aab00f 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_config.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_config.py @@ -231,6 +231,26 @@ def _make_hybrid_kv_cache_config() -> KVCacheConfig: ) +def _mamba_spec() -> MambaSpec: + return MambaSpec( + block_size=16, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + + +def _uniform_spec(spec_kind: str) -> UniformTypeKVCacheSpecs: + # DSA models merge their indexer and MLA layers into a UniformType group, + # a container rather than an AttentionSpec, but still sharded across DCP. + specs: dict[str, Any] = ( + {"mla_layer": _mla_spec(), "indexer_layer": _mla_spec(head_size=128)} + if spec_kind == "attention" + else {"mamba_layer0": _mamba_spec(), "mamba_layer1": _mamba_spec()} + ) + return UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs=specs) + + def _make_mamba_hybrid_kv_cache_config() -> KVCacheConfig: return KVCacheConfig( num_blocks=4, @@ -369,6 +389,61 @@ def test_dcp_scales_attention_but_not_mamba_group_blocks(): ] == [1, 3] +@pytest.mark.parametrize("dcp_size,expected", [(1, 16), (2, 32)]) +def test_dcp_scales_uniform_type_attention_group_blocks(dcp_size, expected): + config = _make_vllm_config( + tensor_parallel_size=2, decode_context_parallel_size=dcp_size + ) + kv_cache_config = KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[ + KVCacheTensor( + size=64, + layers=["mla_layer", "indexer_layer"], + layer_stride=32, + block_stride=8, + ) + ], + kv_cache_groups=[ + KVCacheGroupSpec(["mla_layer", "indexer_layer"], _uniform_spec("attention")) + ], + ) + + offloading_config = build_offloading_config(config, kv_cache_config) + + assert offloading_config.groups[0].tokens_per_block == expected + assert offloading_config.cache.tokens_per_hash == expected + + +@pytest.mark.parametrize( + "spec_kind,expected", + [ + ("attention", 32), + # Mamba state is replicated across DCP ranks, so a uniform group of + # Mamba layers keeps its span. + ("mamba", 16), + ], +) +def test_dcp_scales_uniform_type_group_alongside_mamba(spec_kind, expected): + config = _make_vllm_config(tensor_parallel_size=2, decode_context_parallel_size=2) + config.speculative_config = None + kv_cache_config = KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec(["layer0", "layer1"], _uniform_spec(spec_kind)), + KVCacheGroupSpec(["mamba_layer"], _mamba_spec()), + ], + ) + + offloading_config = build_offloading_config(config, kv_cache_config) + + assert tuple(group.tokens_per_block for group in offloading_config.groups) == ( + expected, + 16, + ) + + def test_preserves_data_parallel_config(): config = _make_vllm_config() config.parallel_config.data_parallel_index = 2 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py index 4fd2b972c7..e5ed896078 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py @@ -4,7 +4,10 @@ from typing import TYPE_CHECKING -from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes +from vllm.v1.core.kv_cache_utils import ( + resolve_dcp_kv_block_size, + resolve_kv_cache_block_sizes, +) from vllm.v1.kv_cache_interface import ( AttentionSpec, FullAttentionSpec, @@ -41,13 +44,9 @@ def build_offloading_config( parallel_config = vllm_config.parallel_config groups = tuple( OffloadingGroupConfig( - tokens_per_block=( - group.kv_cache_spec.block_size - * ( - parallel_config.decode_context_parallel_size - if isinstance(group.kv_cache_spec, AttentionSpec) - else 1 - ) + tokens_per_block=resolve_dcp_kv_block_size( + group.kv_cache_spec, + parallel_config.decode_context_parallel_size, ), layer_names=tuple(group.layer_names), )