[Bugfix][KV Offload] Scale UniformTypeKVCacheSpecs groups by DCP (#50883)

Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
drakosha
2026-09-02 17:11:48 +03:00
committed by GitHub
co-authored by Claude
parent 35faf957d6
commit ba6c60e98f
2 changed files with 82 additions and 8 deletions
@@ -231,6 +231,26 @@ def _make_hybrid_kv_cache_config() -> KVCacheConfig:
)
def _mamba_spec() -> MambaSpec:
return MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
def _uniform_spec(spec_kind: str) -> UniformTypeKVCacheSpecs:
# DSA models merge their indexer and MLA layers into a UniformType group,
# a container rather than an AttentionSpec, but still sharded across DCP.
specs: dict[str, Any] = (
{"mla_layer": _mla_spec(), "indexer_layer": _mla_spec(head_size=128)}
if spec_kind == "attention"
else {"mamba_layer0": _mamba_spec(), "mamba_layer1": _mamba_spec()}
)
return UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs=specs)
def _make_mamba_hybrid_kv_cache_config() -> KVCacheConfig:
return KVCacheConfig(
num_blocks=4,
@@ -369,6 +389,61 @@ def test_dcp_scales_attention_but_not_mamba_group_blocks():
] == [1, 3]
@pytest.mark.parametrize("dcp_size,expected", [(1, 16), (2, 32)])
def test_dcp_scales_uniform_type_attention_group_blocks(dcp_size, expected):
config = _make_vllm_config(
tensor_parallel_size=2, decode_context_parallel_size=dcp_size
)
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(
size=64,
layers=["mla_layer", "indexer_layer"],
layer_stride=32,
block_stride=8,
)
],
kv_cache_groups=[
KVCacheGroupSpec(["mla_layer", "indexer_layer"], _uniform_spec("attention"))
],
)
offloading_config = build_offloading_config(config, kv_cache_config)
assert offloading_config.groups[0].tokens_per_block == expected
assert offloading_config.cache.tokens_per_hash == expected
@pytest.mark.parametrize(
"spec_kind,expected",
[
("attention", 32),
# Mamba state is replicated across DCP ranks, so a uniform group of
# Mamba layers keeps its span.
("mamba", 16),
],
)
def test_dcp_scales_uniform_type_group_alongside_mamba(spec_kind, expected):
config = _make_vllm_config(tensor_parallel_size=2, decode_context_parallel_size=2)
config.speculative_config = None
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["layer0", "layer1"], _uniform_spec(spec_kind)),
KVCacheGroupSpec(["mamba_layer"], _mamba_spec()),
],
)
offloading_config = build_offloading_config(config, kv_cache_config)
assert tuple(group.tokens_per_block for group in offloading_config.groups) == (
expected,
16,
)
def test_preserves_data_parallel_config():
config = _make_vllm_config()
config.parallel_config.data_parallel_index = 2
@@ -4,7 +4,10 @@
from typing import TYPE_CHECKING
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
from vllm.v1.core.kv_cache_utils import (
resolve_dcp_kv_block_size,
resolve_kv_cache_block_sizes,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
FullAttentionSpec,
@@ -41,13 +44,9 @@ def build_offloading_config(
parallel_config = vllm_config.parallel_config
groups = tuple(
OffloadingGroupConfig(
tokens_per_block=(
group.kv_cache_spec.block_size
* (
parallel_config.decode_context_parallel_size
if isinstance(group.kv_cache_spec, AttentionSpec)
else 1
)
tokens_per_block=resolve_dcp_kv_block_size(
group.kv_cache_spec,
parallel_config.decode_context_parallel_size,
),
layer_names=tuple(group.layer_names),
)