[Bugfix][KV Offload] Scale UniformTypeKVCacheSpecs groups by DCP (#50883)

Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
drakosha
2026-09-02 17:11:48 +03:00
committed by GitHub
co-authored by Claude
parent 35faf957d6
commit ba6c60e98f
2 changed files with 82 additions and 8 deletions
@@ -231,6 +231,26 @@ def _make_hybrid_kv_cache_config() -> KVCacheConfig:
)
def _mamba_spec() -> MambaSpec:
return MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
def _uniform_spec(spec_kind: str) -> UniformTypeKVCacheSpecs:
# DSA models merge their indexer and MLA layers into a UniformType group,
# a container rather than an AttentionSpec, but still sharded across DCP.
specs: dict[str, Any] = (
{"mla_layer": _mla_spec(), "indexer_layer": _mla_spec(head_size=128)}
if spec_kind == "attention"
else {"mamba_layer0": _mamba_spec(), "mamba_layer1": _mamba_spec()}
)
return UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs=specs)
def _make_mamba_hybrid_kv_cache_config() -> KVCacheConfig:
return KVCacheConfig(
num_blocks=4,
@@ -369,6 +389,61 @@ def test_dcp_scales_attention_but_not_mamba_group_blocks():
] == [1, 3]
@pytest.mark.parametrize("dcp_size,expected", [(1, 16), (2, 32)])
def test_dcp_scales_uniform_type_attention_group_blocks(dcp_size, expected):
config = _make_vllm_config(
tensor_parallel_size=2, decode_context_parallel_size=dcp_size
)
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(
size=64,
layers=["mla_layer", "indexer_layer"],
layer_stride=32,
block_stride=8,
)
],
kv_cache_groups=[
KVCacheGroupSpec(["mla_layer", "indexer_layer"], _uniform_spec("attention"))
],
)
offloading_config = build_offloading_config(config, kv_cache_config)
assert offloading_config.groups[0].tokens_per_block == expected
assert offloading_config.cache.tokens_per_hash == expected
@pytest.mark.parametrize(
"spec_kind,expected",
[
("attention", 32),
# Mamba state is replicated across DCP ranks, so a uniform group of
# Mamba layers keeps its span.
("mamba", 16),
],
)
def test_dcp_scales_uniform_type_group_alongside_mamba(spec_kind, expected):
config = _make_vllm_config(tensor_parallel_size=2, decode_context_parallel_size=2)
config.speculative_config = None
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["layer0", "layer1"], _uniform_spec(spec_kind)),
KVCacheGroupSpec(["mamba_layer"], _mamba_spec()),
],
)
offloading_config = build_offloading_config(config, kv_cache_config)
assert tuple(group.tokens_per_block for group in offloading_config.groups) == (
expected,
16,
)
def test_preserves_data_parallel_config():
config = _make_vllm_config()
config.parallel_config.data_parallel_index = 2