diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index fdc9198fff..7923e34d0f 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -59,7 +59,7 @@ from vllm.v1.core.kv_cache_utils import ( ) from vllm.v1.hisparse.layout import ( create_hisparse_layout, - get_hisparse_gpu_memory_usage, + get_hisparse_kv_cache_groups, ) from vllm.v1.kv_cache_interface import ( ChunkedLocalAttentionSpec, @@ -118,9 +118,6 @@ def test_hisparse_hma_uses_resolved_gpu_block_size( cache_role=SparseCacheRole.INDEXER, ), } - group_spec = UniformTypeKVCacheSpecs.from_specs(specs) - assert group_spec is not None - group = KVCacheGroupSpec(list(specs), group_spec) config = SimpleNamespace( attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()), model_config=SimpleNamespace( @@ -142,14 +139,9 @@ def test_hisparse_hma_uses_resolved_gpu_block_size( get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC, ), ) - indexer_spec = specs["model.layers.0.self_attn.indexer"] - assert get_hisparse_gpu_memory_usage(config, [group]) == ( - indexer_spec.max_memory_usage_bytes(config) - ) - monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30) cache_config = kv_cache_utils.get_kv_cache_config_from_groups( - config, [group], available_memory=2**30 + config, get_hisparse_kv_cache_groups(config, specs), available_memory=2**30 ) assert cache_config.num_blocks == 7 assert cache_config.hisparse_host_num_blocks is not None @@ -228,8 +220,6 @@ def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok cache_role=SparseCacheRole.INDEXER, ), } - group_spec = UniformTypeKVCacheSpecs.from_specs(specs) - assert group_spec is not None config = SimpleNamespace( attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()), model_config=SimpleNamespace( @@ -240,14 +230,123 @@ def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok ) host_page = specs["model.layers.0.self_attn"].page_size_bytes host_budget = (4 + extra_blocks) * host_page - group = KVCacheGroupSpec(list(specs), group_spec) + groups = get_hisparse_kv_cache_groups(config, specs) if ok: - layout = create_hisparse_layout(config, [group], host_budget=host_budget) + layout = create_hisparse_layout(config, groups, host_budget=host_budget) assert layout.host_num_blocks == 4 + extra_blocks else: with pytest.raises(ValueError, match="increase host_pool_gib"): - create_hisparse_layout(config, [group], host_budget=host_budget) + create_hisparse_layout(config, groups, host_budget=host_budget) + + +@pytest.mark.parametrize( + "max_model_len,num_gpu_blocks,ok", + [ + (8192, 160, False), + (8192, 1000, True), + (32768, 656, False), + (32768, 657, True), + (-1, 657, True), + ], +) +def test_hisparse_pool_must_fit_max_model_len( + monkeypatch, max_model_len, num_gpu_blocks, ok +): + """Each resident group takes its own blocks from HiSparse's shared GPU pool, + so a pool that fits only the indexer pages must be rejected at startup. + Resident pages are capped at the in-flight window: at 32768 tokens with 2048 + in flight, one request needs 512 indexer + 4 x 32 resident + 16 hot blocks, + plus the null block. Auto-fit (-1) must pick a length that passes the same + check.""" + monkeypatch.setattr( + hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True + ) + monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30) + specs: dict[str, KVCacheSpec] = {} + for i in range(4): + specs[f"model.layers.{i}.self_attn"] = MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=576, + dtype=torch.bfloat16, + is_index_group_leader=True, + ) + specs[f"model.layers.{i}.self_attn.indexer"] = MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + cache_role=SparseCacheRole.INDEXER, + ) + config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len)) + config.scheduler_config.max_num_batched_tokens = 2048 + # Async scheduling doubles the in-flight window, and with it the bound. + config.scheduler_config.async_scheduling = False + config.attention_config.hisparse_config = HiSparseConfig() + config.model_config.hf_config.index_topk = 128 + config.cache_config.num_gpu_blocks_override = num_gpu_blocks + config.cache_config.kv_cache_layout = "BLHNC" + # Workers report the resident/hot caches HiSparse derives. + specs |= { + name: group.kv_cache_spec + for group in get_hisparse_kv_cache_groups(config, specs) or [] + if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec)) + for name in group.layer_names + } + + if ok: + kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34]) + else: + with pytest.raises(ValueError, match="max seq len"): + kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34]) + + +def test_hisparse_derived_specs_do_not_affect_kv_cache_layout(monkeypatch): + """Workers report HiSparse's resident/hot caches, which lay out their own raw + backing; KV cache layout resolution must skip them.""" + from vllm.v1.attention.backends.utils import resolve_kv_cache_layout + + monkeypatch.setattr( + hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True + ) + monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False) + specs: dict[str, KVCacheSpec] = { + "model.layers.0.self_attn": MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=576, + dtype=torch.bfloat16, + is_index_group_leader=True, + ), + "model.layers.0.self_attn.indexer": MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + cache_role=SparseCacheRole.INDEXER, + ), + } + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.attention_config.hisparse_config = HiSparseConfig() + config.model_config.hf_config.index_topk = 128 + reported = specs | { + name: group.kv_cache_spec + for group in get_hisparse_kv_cache_groups(config, specs) or [] + if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec)) + for name in group.layer_names + } + assert len(reported) > len(specs) + + layouts = [] + for worker_specs in (specs, reported): + config.cache_config.kv_cache_layout = None + layouts.append( + resolve_kv_cache_layout( + config, [["BLHNC", "LBNHC"]], list(worker_specs.values()) + ) + ) + assert layouts[0] == layouts[1] def test_hisparse_rejects_deepseek_v4(): @@ -260,9 +359,6 @@ def test_hisparse_rejects_deepseek_v4(): model_version="deepseek_v4", ) } - full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs) - assert full_uniform is not None - group = KVCacheGroupSpec(list(full_specs), full_uniform) config = SimpleNamespace( attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()), model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)), @@ -270,11 +366,7 @@ def test_hisparse_rejects_deepseek_v4(): ) with pytest.raises(ValueError, match="does not support DeepSeek V4"): - create_hisparse_layout( - config, - [group], - host_budget=2**30, - ) + get_hisparse_kv_cache_groups(config, full_specs) @pytest.fixture(autouse=True) diff --git a/tests/v1/worker/test_attn_utils.py b/tests/v1/worker/test_attn_utils.py index 2166141638..0e6089d529 100644 --- a/tests/v1/worker/test_attn_utils.py +++ b/tests/v1/worker/test_attn_utils.py @@ -86,6 +86,9 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size( get_attn_backend=lambda backend=backend: backend, ) monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers) + monkeypatch.setattr( + attn_utils_module, "get_hisparse_kv_cache_groups", lambda *_: [] + ) config = SimpleNamespace( attention_config=SimpleNamespace(hisparse_config=object() if enabled else None) ) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index eb28fd1e78..529d816c06 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -291,7 +291,10 @@ def resolve_kv_cache_layout( # specs can re-interpret HNC with different sizes as long as the total number of # bytes is the same. If not block-compact, each spec must agree on HNC to alias # the same page (this aliasing is done by the Hybrid Memory Allocator, HMA). - kv_cache_specs = tuple(kv_cache_specs or ()) + # Specs without per-layer views lay out their own raw backing tensor. + kv_cache_specs = tuple( + spec for spec in kv_cache_specs or () if spec.has_layer_views + ) hnc_shapes = { (spec.num_heads, spec.num_states, spec.page_size_bytes) for spec in kv_cache_specs diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 4c8b807d16..46934d530b 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -21,7 +21,6 @@ from vllm.utils.math_utils import cdiv, round_up from vllm.utils.mem_utils import format_gib from vllm.utils.torch_utils import get_dtype_size from vllm.v1.hisparse.layout import ( - get_hisparse_gpu_memory_usage, get_hisparse_host_pool_bytes, get_hisparse_kv_cache_config, get_hisparse_kv_cache_groups, @@ -1155,7 +1154,9 @@ def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int: `available_memory` into `num_blocks`. Used to compute the effective KV cache capacity once `num_gpu_blocks_override` is applied. """ - return _get_kv_cache_bytes_per_block(kv_cache_groups) + return _get_kv_cache_bytes_per_block( + [group for group in kv_cache_groups if not group.host_resident] + ) def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int: @@ -2500,12 +2501,10 @@ def _max_memory_usage_bytes_from_groups( Each group independently claims blocks from the shared pool, so a request consumes the sum of the per-group block counts, i.e. ``bytes_per_block * total_blocks``. """ + kv_cache_groups = [group for group in kv_cache_groups if not group.host_resident] if not kv_cache_groups: return 0 - if vllm_config.attention_config.hisparse_config is not None: - return get_hisparse_gpu_memory_usage(vllm_config, kv_cache_groups) - if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None: ( attn_group, @@ -2563,12 +2562,11 @@ def _estimate_max_model_len_from_groups( vllm_config.model_config.max_model_len = model_len if hisparse_enabled: try: - config = get_kv_cache_config_from_groups( + get_kv_cache_config_from_groups( vllm_config, kv_cache_groups, available_memory ) except ValueError: return False - return get_max_concurrency_for_kv_cache_config(vllm_config, config) >= 1 return ( _max_memory_usage_bytes_from_groups(vllm_config, kv_cache_groups) <= available_memory @@ -2678,6 +2676,9 @@ def _project_kv_cache_groups_to_worker( worker_layer_names = [ layer_name for layer_name in group.layer_names if layer_name in worker_spec ] + if len(worker_layer_names) == len(group.layer_names): + projected_groups.append(group) + continue group_spec = group.kv_cache_spec if worker_layer_names and isinstance(group_spec, UniformTypeKVCacheSpecs): group_spec = UniformTypeKVCacheSpecs( @@ -2793,9 +2794,6 @@ def get_kv_cache_configs( adjusted_memory.append(override * bytes_per_block) available_memory = adjusted_memory - if vllm_config.attention_config.hisparse_config is not None: - available_memory = [min(available_memory)] * len(available_memory) - # Reserve the null block BlockPool permanently holds back, so auto-fit and # the capacity check both plan against usable blocks. Allocation below # still uses the full memory. diff --git a/vllm/v1/hisparse/binding.py b/vllm/v1/hisparse/binding.py index 716211fbb3..88c8976b56 100644 --- a/vllm/v1/hisparse/binding.py +++ b/vllm/v1/hisparse/binding.py @@ -12,6 +12,7 @@ from vllm.config import VllmConfig from vllm.v1.hisparse.layout import ( HISPARSE_HOT_SUFFIX, HISPARSE_RESIDENT_SUFFIX, + get_hisparse_kv_cache_groups, ) from vllm.v1.hisparse.runtime import ( HiSparseCacheHandle, @@ -36,21 +37,20 @@ if TYPE_CHECKING: from vllm.v1.worker.gpu.block_table import BlockTables -def resolve_hisparse_block_size( +def resolve_hisparse_specs( vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec], attn_layers: Mapping[str, "AttentionLayerBase"], -) -> None: - """Resolve a common kernel block size in-place for HiSparse MLA specs.""" - if vllm_config.attention_config.hisparse_config is None: - return +) -> dict[str, KVCacheSpec]: + """Resolve a common kernel block size for HiSparse MLA specs, and add the + resident/hot specs HiSparse derives from them.""" mla_specs = { name: spec for name, spec in kv_cache_spec.items() if isinstance(spec, MLAAttentionSpec) } if not mla_specs: - return + return kv_cache_spec block_sizes = {spec.block_size for spec in mla_specs.values()} if len(block_sizes) != 1: raise ValueError("HiSparse requires one scheduler block size.") @@ -62,10 +62,18 @@ def resolve_hisparse_block_size( "HiSparse requires a GPU block size supported by every sparse " f"attention and indexer backend: {error}" ) from error - kv_cache_spec.update( - (name, spec.copy_with_new_block_size(block_size)) + kv_cache_spec = kv_cache_spec | { + name: spec.copy_with_new_block_size(block_size) for name, spec in mla_specs.items() - ) + } + groups = get_hisparse_kv_cache_groups(vllm_config, kv_cache_spec) + assert groups is not None + return kv_cache_spec | { + layer_name: group.kv_cache_spec + for group in groups + if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec)) + for layer_name in group.layer_names + } def allocate_hisparse_kv_caches( diff --git a/vllm/v1/hisparse/layout.py b/vllm/v1/hisparse/layout.py index e177768fc9..8ebc63c0ed 100644 --- a/vllm/v1/hisparse/layout.py +++ b/vllm/v1/hisparse/layout.py @@ -47,6 +47,10 @@ class HiSparseLayout: def get_hisparse_kv_cache_groups( vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec] ) -> list[KVCacheGroupSpec] | None: + """The groups HiSparse allocates: the host source group, then the GPU + groups sharing one device pool. Derived resident/hot specs in + `kv_cache_spec` (see `resolve_hisparse_specs`) are laid out again + from the attention specs they belong to.""" attention_config = getattr(vllm_config, "attention_config", None) if attention_config is None or attention_config.hisparse_config is None: return None @@ -59,9 +63,11 @@ def get_hisparse_kv_cache_groups( other_specs = { name: spec for name, spec in kv_cache_spec.items() - if not isinstance(spec, MLAAttentionSpec) + if not isinstance( + spec, (MLAAttentionSpec, HiSparseResidentSpec, HiSparseHotSpec) + ) } - if not mla_specs or not other_specs: + if not mla_specs: return None from vllm.v1.core.kv_cache_utils import get_kv_cache_groups @@ -69,7 +75,10 @@ def get_hisparse_kv_cache_groups( mla_group_spec = UniformTypeKVCacheSpecs.from_specs(mla_specs) assert mla_group_spec is not None mla_group = KVCacheGroupSpec(list(mla_specs), mla_group_spec) - return [mla_group, *get_kv_cache_groups(vllm_config, other_specs)] + regular_groups = ( + get_kv_cache_groups(vllm_config, other_specs) if other_specs else [] + ) + return _lay_out_hisparse_groups(vllm_config, [mla_group, *regular_groups]) def get_hisparse_host_pool_bytes(vllm_config: VllmConfig) -> int: @@ -112,24 +121,9 @@ def _partition_hisparse_specs( return source_specs, indexer_specs -def get_hisparse_gpu_memory_usage( - vllm_config: VllmConfig, - kv_cache_groups: list[KVCacheGroupSpec], -) -> int: - _, indexer_specs = _partition_hisparse_specs(kv_cache_groups) - return sum( - spec.max_memory_usage_bytes(vllm_config) for spec in indexer_specs.values() - ) + sum( - group.kv_cache_spec.max_memory_usage_bytes(vllm_config) - for group in kv_cache_groups[1:] - ) - - -def create_hisparse_layout( - vllm_config: VllmConfig, - groups: list[KVCacheGroupSpec], - host_budget: int, -) -> HiSparseLayout: +def _lay_out_hisparse_groups( + vllm_config: VllmConfig, groups: list[KVCacheGroupSpec] +) -> list[KVCacheGroupSpec]: source_specs, indexer_specs = _partition_hisparse_specs(groups) block_sizes = { spec.block_size for spec in (*source_specs.values(), *indexer_specs.values()) @@ -225,11 +219,25 @@ def create_hisparse_layout( enable_kv_transfer=True, ) regular_groups = groups[1:] - gpu_groups = [indexer_group, *resident_groups, *hot_groups, *regular_groups] + return [ + source_group, + indexer_group, + *resident_groups, + *hot_groups, + *regular_groups, + ] + +def create_hisparse_layout( + vllm_config: VllmConfig, + groups: list[KVCacheGroupSpec], + host_budget: int, +) -> HiSparseLayout: + """Size the host pool for groups laid out by `get_hisparse_kv_cache_groups`.""" + (source_group,) = [group for group in groups if group.host_resident] shared_host_pool = use_shared_hisparse_host_pool(vllm_config) host_block_stride = get_hisparse_host_block_stride( - sum(spec.page_size_bytes for spec in source_specs.values()), + source_group.kv_cache_spec.page_size_bytes, use_shared_host_pool=shared_host_pool, ) host_num_blocks = host_budget // host_block_stride @@ -237,6 +245,7 @@ def create_hisparse_layout( raise ValueError("HiSparse has no allocatable host blocks.") # Every computed page needs a host block, so one request at max_model_len # must fit alongside the pool's null block and a copy-on-write tail. + gpu_block_size = source_group.kv_cache_spec.block_size min_host_blocks = cdiv(vllm_config.model_config.max_model_len, gpu_block_size) + 2 if host_num_blocks < min_host_blocks: raise ValueError( @@ -246,7 +255,7 @@ def create_hisparse_layout( return HiSparseLayout( source_group=source_group, - device_groups=gpu_groups, + device_groups=[group for group in groups if not group.host_resident], host_num_blocks=host_num_blocks, host_block_stride=host_block_stride, shared_host_pool=shared_host_pool, diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 248f03dd02..846e6bcad0 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -472,9 +472,11 @@ class HiSparseResidentSpec(KVCacheSpec): return cdiv(num_tokens, self.block_size) def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: - return cdiv(vllm_config.model_config.max_model_len, self.block_size) * ( - self.page_size + max_blocks = self.max_admission_blocks_per_request( + max_in_flight_tokens=vllm_config.max_in_flight_tokens, + max_model_len=vllm_config.model_config.max_model_len, ) + return max_blocks * self.page_size @property def has_layer_views(self) -> bool: diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 19de125a9b..9e9ffd7be1 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -23,7 +23,7 @@ from vllm.v1.attention.backend import ( from vllm.v1.attention.backends.utils import create_fast_prefill_custom_backend from vllm.v1.hisparse.binding import ( init_hisparse_kv_cache, - resolve_hisparse_block_size, + resolve_hisparse_specs, ) from vllm.v1.kv_cache_interface import ( AttentionSpec, @@ -142,7 +142,8 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]: if isinstance(spec, AttentionSpec): spec = attn_module.get_attn_backend().customize_spec(spec) kv_cache_spec[layer_name] = spec - resolve_hisparse_block_size(vllm_config, kv_cache_spec, attn_layers) + if vllm_config.attention_config.hisparse_config is not None: + kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers) return kv_cache_spec