[Bugfix][HiSparse] Size the KV cache from the groups HiSparse allocates (#59450)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-10-02 04:11:55 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Lucas Wilkinson
parent 1a9eaa3a68
commit f7999d2e44
8 changed files with 187 additions and 71 deletions
+115 -23
View File
@@ -59,7 +59,7 @@ from vllm.v1.core.kv_cache_utils import (
)
from vllm.v1.hisparse.layout import (
create_hisparse_layout,
get_hisparse_gpu_memory_usage,
get_hisparse_kv_cache_groups,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
@@ -118,9 +118,6 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
cache_role=SparseCacheRole.INDEXER,
),
}
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
assert group_spec is not None
group = KVCacheGroupSpec(list(specs), group_spec)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(
@@ -142,14 +139,9 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC,
),
)
indexer_spec = specs["model.layers.0.self_attn.indexer"]
assert get_hisparse_gpu_memory_usage(config, [group]) == (
indexer_spec.max_memory_usage_bytes(config)
)
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
config, [group], available_memory=2**30
config, get_hisparse_kv_cache_groups(config, specs), available_memory=2**30
)
assert cache_config.num_blocks == 7
assert cache_config.hisparse_host_num_blocks is not None
@@ -228,8 +220,6 @@ def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok
cache_role=SparseCacheRole.INDEXER,
),
}
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
assert group_spec is not None
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(
@@ -240,14 +230,123 @@ def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok
)
host_page = specs["model.layers.0.self_attn"].page_size_bytes
host_budget = (4 + extra_blocks) * host_page
group = KVCacheGroupSpec(list(specs), group_spec)
groups = get_hisparse_kv_cache_groups(config, specs)
if ok:
layout = create_hisparse_layout(config, [group], host_budget=host_budget)
layout = create_hisparse_layout(config, groups, host_budget=host_budget)
assert layout.host_num_blocks == 4 + extra_blocks
else:
with pytest.raises(ValueError, match="increase host_pool_gib"):
create_hisparse_layout(config, [group], host_budget=host_budget)
create_hisparse_layout(config, groups, host_budget=host_budget)
@pytest.mark.parametrize(
"max_model_len,num_gpu_blocks,ok",
[
(8192, 160, False),
(8192, 1000, True),
(32768, 656, False),
(32768, 657, True),
(-1, 657, True),
],
)
def test_hisparse_pool_must_fit_max_model_len(
monkeypatch, max_model_len, num_gpu_blocks, ok
):
"""Each resident group takes its own blocks from HiSparse's shared GPU pool,
so a pool that fits only the indexer pages must be rejected at startup.
Resident pages are capped at the in-flight window: at 32768 tokens with 2048
in flight, one request needs 512 indexer + 4 x 32 resident + 16 hot blocks,
plus the null block. Auto-fit (-1) must pick a length that passes the same
check."""
monkeypatch.setattr(
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
)
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
specs: dict[str, KVCacheSpec] = {}
for i in range(4):
specs[f"model.layers.{i}.self_attn"] = MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
is_index_group_leader=True,
)
specs[f"model.layers.{i}.self_attn.indexer"] = MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
)
config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
config.scheduler_config.max_num_batched_tokens = 2048
# Async scheduling doubles the in-flight window, and with it the bound.
config.scheduler_config.async_scheduling = False
config.attention_config.hisparse_config = HiSparseConfig()
config.model_config.hf_config.index_topk = 128
config.cache_config.num_gpu_blocks_override = num_gpu_blocks
config.cache_config.kv_cache_layout = "BLHNC"
# Workers report the resident/hot caches HiSparse derives.
specs |= {
name: group.kv_cache_spec
for group in get_hisparse_kv_cache_groups(config, specs) or []
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
for name in group.layer_names
}
if ok:
kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34])
else:
with pytest.raises(ValueError, match="max seq len"):
kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34])
def test_hisparse_derived_specs_do_not_affect_kv_cache_layout(monkeypatch):
"""Workers report HiSparse's resident/hot caches, which lay out their own raw
backing; KV cache layout resolution must skip them."""
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
monkeypatch.setattr(
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
)
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
specs: dict[str, KVCacheSpec] = {
"model.layers.0.self_attn": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
is_index_group_leader=True,
),
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
),
}
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.attention_config.hisparse_config = HiSparseConfig()
config.model_config.hf_config.index_topk = 128
reported = specs | {
name: group.kv_cache_spec
for group in get_hisparse_kv_cache_groups(config, specs) or []
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
for name in group.layer_names
}
assert len(reported) > len(specs)
layouts = []
for worker_specs in (specs, reported):
config.cache_config.kv_cache_layout = None
layouts.append(
resolve_kv_cache_layout(
config, [["BLHNC", "LBNHC"]], list(worker_specs.values())
)
)
assert layouts[0] == layouts[1]
def test_hisparse_rejects_deepseek_v4():
@@ -260,9 +359,6 @@ def test_hisparse_rejects_deepseek_v4():
model_version="deepseek_v4",
)
}
full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs)
assert full_uniform is not None
group = KVCacheGroupSpec(list(full_specs), full_uniform)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)),
@@ -270,11 +366,7 @@ def test_hisparse_rejects_deepseek_v4():
)
with pytest.raises(ValueError, match="does not support DeepSeek V4"):
create_hisparse_layout(
config,
[group],
host_budget=2**30,
)
get_hisparse_kv_cache_groups(config, full_specs)
@pytest.fixture(autouse=True)
+3
View File
@@ -86,6 +86,9 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
get_attn_backend=lambda backend=backend: backend,
)
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
monkeypatch.setattr(
attn_utils_module, "get_hisparse_kv_cache_groups", lambda *_: []
)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=object() if enabled else None)
)
+4 -1
View File
@@ -291,7 +291,10 @@ def resolve_kv_cache_layout(
# specs can re-interpret HNC with different sizes as long as the total number of
# bytes is the same. If not block-compact, each spec must agree on HNC to alias
# the same page (this aliasing is done by the Hybrid Memory Allocator, HMA).
kv_cache_specs = tuple(kv_cache_specs or ())
# Specs without per-layer views lay out their own raw backing tensor.
kv_cache_specs = tuple(
spec for spec in kv_cache_specs or () if spec.has_layer_views
)
hnc_shapes = {
(spec.num_heads, spec.num_states, spec.page_size_bytes)
for spec in kv_cache_specs
+8 -10
View File
@@ -21,7 +21,6 @@ from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.mem_utils import format_gib
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.hisparse.layout import (
get_hisparse_gpu_memory_usage,
get_hisparse_host_pool_bytes,
get_hisparse_kv_cache_config,
get_hisparse_kv_cache_groups,
@@ -1155,7 +1154,9 @@ def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
`available_memory` into `num_blocks`. Used to compute the effective KV cache
capacity once `num_gpu_blocks_override` is applied.
"""
return _get_kv_cache_bytes_per_block(kv_cache_groups)
return _get_kv_cache_bytes_per_block(
[group for group in kv_cache_groups if not group.host_resident]
)
def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
@@ -2500,12 +2501,10 @@ def _max_memory_usage_bytes_from_groups(
Each group independently claims blocks from the shared pool, so a request consumes
the sum of the per-group block counts, i.e. ``bytes_per_block * total_blocks``.
"""
kv_cache_groups = [group for group in kv_cache_groups if not group.host_resident]
if not kv_cache_groups:
return 0
if vllm_config.attention_config.hisparse_config is not None:
return get_hisparse_gpu_memory_usage(vllm_config, kv_cache_groups)
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
(
attn_group,
@@ -2563,12 +2562,11 @@ def _estimate_max_model_len_from_groups(
vllm_config.model_config.max_model_len = model_len
if hisparse_enabled:
try:
config = get_kv_cache_config_from_groups(
get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, available_memory
)
except ValueError:
return False
return get_max_concurrency_for_kv_cache_config(vllm_config, config) >= 1
return (
_max_memory_usage_bytes_from_groups(vllm_config, kv_cache_groups)
<= available_memory
@@ -2678,6 +2676,9 @@ def _project_kv_cache_groups_to_worker(
worker_layer_names = [
layer_name for layer_name in group.layer_names if layer_name in worker_spec
]
if len(worker_layer_names) == len(group.layer_names):
projected_groups.append(group)
continue
group_spec = group.kv_cache_spec
if worker_layer_names and isinstance(group_spec, UniformTypeKVCacheSpecs):
group_spec = UniformTypeKVCacheSpecs(
@@ -2793,9 +2794,6 @@ def get_kv_cache_configs(
adjusted_memory.append(override * bytes_per_block)
available_memory = adjusted_memory
if vllm_config.attention_config.hisparse_config is not None:
available_memory = [min(available_memory)] * len(available_memory)
# Reserve the null block BlockPool permanently holds back, so auto-fit and
# the capacity check both plan against usable blocks. Allocation below
# still uses the full memory.
+17 -9
View File
@@ -12,6 +12,7 @@ from vllm.config import VllmConfig
from vllm.v1.hisparse.layout import (
HISPARSE_HOT_SUFFIX,
HISPARSE_RESIDENT_SUFFIX,
get_hisparse_kv_cache_groups,
)
from vllm.v1.hisparse.runtime import (
HiSparseCacheHandle,
@@ -36,21 +37,20 @@ if TYPE_CHECKING:
from vllm.v1.worker.gpu.block_table import BlockTables
def resolve_hisparse_block_size(
def resolve_hisparse_specs(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
attn_layers: Mapping[str, "AttentionLayerBase"],
) -> None:
"""Resolve a common kernel block size in-place for HiSparse MLA specs."""
if vllm_config.attention_config.hisparse_config is None:
return
) -> dict[str, KVCacheSpec]:
"""Resolve a common kernel block size for HiSparse MLA specs, and add the
resident/hot specs HiSparse derives from them."""
mla_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if isinstance(spec, MLAAttentionSpec)
}
if not mla_specs:
return
return kv_cache_spec
block_sizes = {spec.block_size for spec in mla_specs.values()}
if len(block_sizes) != 1:
raise ValueError("HiSparse requires one scheduler block size.")
@@ -62,10 +62,18 @@ def resolve_hisparse_block_size(
"HiSparse requires a GPU block size supported by every sparse "
f"attention and indexer backend: {error}"
) from error
kv_cache_spec.update(
(name, spec.copy_with_new_block_size(block_size))
kv_cache_spec = kv_cache_spec | {
name: spec.copy_with_new_block_size(block_size)
for name, spec in mla_specs.items()
)
}
groups = get_hisparse_kv_cache_groups(vllm_config, kv_cache_spec)
assert groups is not None
return kv_cache_spec | {
layer_name: group.kv_cache_spec
for group in groups
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
for layer_name in group.layer_names
}
def allocate_hisparse_kv_caches(
+33 -24
View File
@@ -47,6 +47,10 @@ class HiSparseLayout:
def get_hisparse_kv_cache_groups(
vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec]
) -> list[KVCacheGroupSpec] | None:
"""The groups HiSparse allocates: the host source group, then the GPU
groups sharing one device pool. Derived resident/hot specs in
`kv_cache_spec` (see `resolve_hisparse_specs`) are laid out again
from the attention specs they belong to."""
attention_config = getattr(vllm_config, "attention_config", None)
if attention_config is None or attention_config.hisparse_config is None:
return None
@@ -59,9 +63,11 @@ def get_hisparse_kv_cache_groups(
other_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if not isinstance(spec, MLAAttentionSpec)
if not isinstance(
spec, (MLAAttentionSpec, HiSparseResidentSpec, HiSparseHotSpec)
)
}
if not mla_specs or not other_specs:
if not mla_specs:
return None
from vllm.v1.core.kv_cache_utils import get_kv_cache_groups
@@ -69,7 +75,10 @@ def get_hisparse_kv_cache_groups(
mla_group_spec = UniformTypeKVCacheSpecs.from_specs(mla_specs)
assert mla_group_spec is not None
mla_group = KVCacheGroupSpec(list(mla_specs), mla_group_spec)
return [mla_group, *get_kv_cache_groups(vllm_config, other_specs)]
regular_groups = (
get_kv_cache_groups(vllm_config, other_specs) if other_specs else []
)
return _lay_out_hisparse_groups(vllm_config, [mla_group, *regular_groups])
def get_hisparse_host_pool_bytes(vllm_config: VllmConfig) -> int:
@@ -112,24 +121,9 @@ def _partition_hisparse_specs(
return source_specs, indexer_specs
def get_hisparse_gpu_memory_usage(
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
) -> int:
_, indexer_specs = _partition_hisparse_specs(kv_cache_groups)
return sum(
spec.max_memory_usage_bytes(vllm_config) for spec in indexer_specs.values()
) + sum(
group.kv_cache_spec.max_memory_usage_bytes(vllm_config)
for group in kv_cache_groups[1:]
)
def create_hisparse_layout(
vllm_config: VllmConfig,
groups: list[KVCacheGroupSpec],
host_budget: int,
) -> HiSparseLayout:
def _lay_out_hisparse_groups(
vllm_config: VllmConfig, groups: list[KVCacheGroupSpec]
) -> list[KVCacheGroupSpec]:
source_specs, indexer_specs = _partition_hisparse_specs(groups)
block_sizes = {
spec.block_size for spec in (*source_specs.values(), *indexer_specs.values())
@@ -225,11 +219,25 @@ def create_hisparse_layout(
enable_kv_transfer=True,
)
regular_groups = groups[1:]
gpu_groups = [indexer_group, *resident_groups, *hot_groups, *regular_groups]
return [
source_group,
indexer_group,
*resident_groups,
*hot_groups,
*regular_groups,
]
def create_hisparse_layout(
vllm_config: VllmConfig,
groups: list[KVCacheGroupSpec],
host_budget: int,
) -> HiSparseLayout:
"""Size the host pool for groups laid out by `get_hisparse_kv_cache_groups`."""
(source_group,) = [group for group in groups if group.host_resident]
shared_host_pool = use_shared_hisparse_host_pool(vllm_config)
host_block_stride = get_hisparse_host_block_stride(
sum(spec.page_size_bytes for spec in source_specs.values()),
source_group.kv_cache_spec.page_size_bytes,
use_shared_host_pool=shared_host_pool,
)
host_num_blocks = host_budget // host_block_stride
@@ -237,6 +245,7 @@ def create_hisparse_layout(
raise ValueError("HiSparse has no allocatable host blocks.")
# Every computed page needs a host block, so one request at max_model_len
# must fit alongside the pool's null block and a copy-on-write tail.
gpu_block_size = source_group.kv_cache_spec.block_size
min_host_blocks = cdiv(vllm_config.model_config.max_model_len, gpu_block_size) + 2
if host_num_blocks < min_host_blocks:
raise ValueError(
@@ -246,7 +255,7 @@ def create_hisparse_layout(
return HiSparseLayout(
source_group=source_group,
device_groups=gpu_groups,
device_groups=[group for group in groups if not group.host_resident],
host_num_blocks=host_num_blocks,
host_block_stride=host_block_stride,
shared_host_pool=shared_host_pool,
+4 -2
View File
@@ -472,9 +472,11 @@ class HiSparseResidentSpec(KVCacheSpec):
return cdiv(num_tokens, self.block_size)
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
return cdiv(vllm_config.model_config.max_model_len, self.block_size) * (
self.page_size
max_blocks = self.max_admission_blocks_per_request(
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
max_model_len=vllm_config.model_config.max_model_len,
)
return max_blocks * self.page_size
@property
def has_layer_views(self) -> bool:
+3 -2
View File
@@ -23,7 +23,7 @@ from vllm.v1.attention.backend import (
from vllm.v1.attention.backends.utils import create_fast_prefill_custom_backend
from vllm.v1.hisparse.binding import (
init_hisparse_kv_cache,
resolve_hisparse_block_size,
resolve_hisparse_specs,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
@@ -142,7 +142,8 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
if isinstance(spec, AttentionSpec):
spec = attn_module.get_attn_backend().customize_spec(spec)
kv_cache_spec[layer_name] = spec
resolve_hisparse_block_size(vllm_config, kv_cache_spec, attn_layers)
if vllm_config.attention_config.hisparse_config is not None:
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
return kv_cache_spec