[Bugfix][HiSparse] Size the KV cache from the groups HiSparse allocates (#59450)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Lucas Wilkinson
parent
1a9eaa3a68
commit
f7999d2e44
@@ -59,7 +59,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
)
|
||||
from vllm.v1.hisparse.layout import (
|
||||
create_hisparse_layout,
|
||||
get_hisparse_gpu_memory_usage,
|
||||
get_hisparse_kv_cache_groups,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
@@ -118,9 +118,6 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
}
|
||||
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
|
||||
assert group_spec is not None
|
||||
group = KVCacheGroupSpec(list(specs), group_spec)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(
|
||||
@@ -142,14 +139,9 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC,
|
||||
),
|
||||
)
|
||||
indexer_spec = specs["model.layers.0.self_attn.indexer"]
|
||||
assert get_hisparse_gpu_memory_usage(config, [group]) == (
|
||||
indexer_spec.max_memory_usage_bytes(config)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, [group], available_memory=2**30
|
||||
config, get_hisparse_kv_cache_groups(config, specs), available_memory=2**30
|
||||
)
|
||||
assert cache_config.num_blocks == 7
|
||||
assert cache_config.hisparse_host_num_blocks is not None
|
||||
@@ -228,8 +220,6 @@ def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
}
|
||||
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
|
||||
assert group_spec is not None
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(
|
||||
@@ -240,14 +230,123 @@ def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok
|
||||
)
|
||||
host_page = specs["model.layers.0.self_attn"].page_size_bytes
|
||||
host_budget = (4 + extra_blocks) * host_page
|
||||
group = KVCacheGroupSpec(list(specs), group_spec)
|
||||
groups = get_hisparse_kv_cache_groups(config, specs)
|
||||
|
||||
if ok:
|
||||
layout = create_hisparse_layout(config, [group], host_budget=host_budget)
|
||||
layout = create_hisparse_layout(config, groups, host_budget=host_budget)
|
||||
assert layout.host_num_blocks == 4 + extra_blocks
|
||||
else:
|
||||
with pytest.raises(ValueError, match="increase host_pool_gib"):
|
||||
create_hisparse_layout(config, [group], host_budget=host_budget)
|
||||
create_hisparse_layout(config, groups, host_budget=host_budget)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"max_model_len,num_gpu_blocks,ok",
|
||||
[
|
||||
(8192, 160, False),
|
||||
(8192, 1000, True),
|
||||
(32768, 656, False),
|
||||
(32768, 657, True),
|
||||
(-1, 657, True),
|
||||
],
|
||||
)
|
||||
def test_hisparse_pool_must_fit_max_model_len(
|
||||
monkeypatch, max_model_len, num_gpu_blocks, ok
|
||||
):
|
||||
"""Each resident group takes its own blocks from HiSparse's shared GPU pool,
|
||||
so a pool that fits only the indexer pages must be rejected at startup.
|
||||
Resident pages are capped at the in-flight window: at 32768 tokens with 2048
|
||||
in flight, one request needs 512 indexer + 4 x 32 resident + 16 hot blocks,
|
||||
plus the null block. Auto-fit (-1) must pick a length that passes the same
|
||||
check."""
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for i in range(4):
|
||||
specs[f"model.layers.{i}.self_attn"] = MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
is_index_group_leader=True,
|
||||
)
|
||||
specs[f"model.layers.{i}.self_attn.indexer"] = MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
)
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
|
||||
config.scheduler_config.max_num_batched_tokens = 2048
|
||||
# Async scheduling doubles the in-flight window, and with it the bound.
|
||||
config.scheduler_config.async_scheduling = False
|
||||
config.attention_config.hisparse_config = HiSparseConfig()
|
||||
config.model_config.hf_config.index_topk = 128
|
||||
config.cache_config.num_gpu_blocks_override = num_gpu_blocks
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
# Workers report the resident/hot caches HiSparse derives.
|
||||
specs |= {
|
||||
name: group.kv_cache_spec
|
||||
for group in get_hisparse_kv_cache_groups(config, specs) or []
|
||||
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
|
||||
for name in group.layer_names
|
||||
}
|
||||
|
||||
if ok:
|
||||
kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34])
|
||||
else:
|
||||
with pytest.raises(ValueError, match="max seq len"):
|
||||
kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34])
|
||||
|
||||
|
||||
def test_hisparse_derived_specs_do_not_affect_kv_cache_layout(monkeypatch):
|
||||
"""Workers report HiSparse's resident/hot caches, which lay out their own raw
|
||||
backing; KV cache layout resolution must skip them."""
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
|
||||
specs: dict[str, KVCacheSpec] = {
|
||||
"model.layers.0.self_attn": MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
}
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.attention_config.hisparse_config = HiSparseConfig()
|
||||
config.model_config.hf_config.index_topk = 128
|
||||
reported = specs | {
|
||||
name: group.kv_cache_spec
|
||||
for group in get_hisparse_kv_cache_groups(config, specs) or []
|
||||
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
|
||||
for name in group.layer_names
|
||||
}
|
||||
assert len(reported) > len(specs)
|
||||
|
||||
layouts = []
|
||||
for worker_specs in (specs, reported):
|
||||
config.cache_config.kv_cache_layout = None
|
||||
layouts.append(
|
||||
resolve_kv_cache_layout(
|
||||
config, [["BLHNC", "LBNHC"]], list(worker_specs.values())
|
||||
)
|
||||
)
|
||||
assert layouts[0] == layouts[1]
|
||||
|
||||
|
||||
def test_hisparse_rejects_deepseek_v4():
|
||||
@@ -260,9 +359,6 @@ def test_hisparse_rejects_deepseek_v4():
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
}
|
||||
full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs)
|
||||
assert full_uniform is not None
|
||||
group = KVCacheGroupSpec(list(full_specs), full_uniform)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)),
|
||||
@@ -270,11 +366,7 @@ def test_hisparse_rejects_deepseek_v4():
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not support DeepSeek V4"):
|
||||
create_hisparse_layout(
|
||||
config,
|
||||
[group],
|
||||
host_budget=2**30,
|
||||
)
|
||||
get_hisparse_kv_cache_groups(config, full_specs)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -86,6 +86,9 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
get_attn_backend=lambda backend=backend: backend,
|
||||
)
|
||||
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
|
||||
monkeypatch.setattr(
|
||||
attn_utils_module, "get_hisparse_kv_cache_groups", lambda *_: []
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=object() if enabled else None)
|
||||
)
|
||||
|
||||
@@ -291,7 +291,10 @@ def resolve_kv_cache_layout(
|
||||
# specs can re-interpret HNC with different sizes as long as the total number of
|
||||
# bytes is the same. If not block-compact, each spec must agree on HNC to alias
|
||||
# the same page (this aliasing is done by the Hybrid Memory Allocator, HMA).
|
||||
kv_cache_specs = tuple(kv_cache_specs or ())
|
||||
# Specs without per-layer views lay out their own raw backing tensor.
|
||||
kv_cache_specs = tuple(
|
||||
spec for spec in kv_cache_specs or () if spec.has_layer_views
|
||||
)
|
||||
hnc_shapes = {
|
||||
(spec.num_heads, spec.num_states, spec.page_size_bytes)
|
||||
for spec in kv_cache_specs
|
||||
|
||||
@@ -21,7 +21,6 @@ from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.hisparse.layout import (
|
||||
get_hisparse_gpu_memory_usage,
|
||||
get_hisparse_host_pool_bytes,
|
||||
get_hisparse_kv_cache_config,
|
||||
get_hisparse_kv_cache_groups,
|
||||
@@ -1155,7 +1154,9 @@ def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
|
||||
`available_memory` into `num_blocks`. Used to compute the effective KV cache
|
||||
capacity once `num_gpu_blocks_override` is applied.
|
||||
"""
|
||||
return _get_kv_cache_bytes_per_block(kv_cache_groups)
|
||||
return _get_kv_cache_bytes_per_block(
|
||||
[group for group in kv_cache_groups if not group.host_resident]
|
||||
)
|
||||
|
||||
|
||||
def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
|
||||
@@ -2500,12 +2501,10 @@ def _max_memory_usage_bytes_from_groups(
|
||||
Each group independently claims blocks from the shared pool, so a request consumes
|
||||
the sum of the per-group block counts, i.e. ``bytes_per_block * total_blocks``.
|
||||
"""
|
||||
kv_cache_groups = [group for group in kv_cache_groups if not group.host_resident]
|
||||
if not kv_cache_groups:
|
||||
return 0
|
||||
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
return get_hisparse_gpu_memory_usage(vllm_config, kv_cache_groups)
|
||||
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
(
|
||||
attn_group,
|
||||
@@ -2563,12 +2562,11 @@ def _estimate_max_model_len_from_groups(
|
||||
vllm_config.model_config.max_model_len = model_len
|
||||
if hisparse_enabled:
|
||||
try:
|
||||
config = get_kv_cache_config_from_groups(
|
||||
get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, available_memory
|
||||
)
|
||||
except ValueError:
|
||||
return False
|
||||
return get_max_concurrency_for_kv_cache_config(vllm_config, config) >= 1
|
||||
return (
|
||||
_max_memory_usage_bytes_from_groups(vllm_config, kv_cache_groups)
|
||||
<= available_memory
|
||||
@@ -2678,6 +2676,9 @@ def _project_kv_cache_groups_to_worker(
|
||||
worker_layer_names = [
|
||||
layer_name for layer_name in group.layer_names if layer_name in worker_spec
|
||||
]
|
||||
if len(worker_layer_names) == len(group.layer_names):
|
||||
projected_groups.append(group)
|
||||
continue
|
||||
group_spec = group.kv_cache_spec
|
||||
if worker_layer_names and isinstance(group_spec, UniformTypeKVCacheSpecs):
|
||||
group_spec = UniformTypeKVCacheSpecs(
|
||||
@@ -2793,9 +2794,6 @@ def get_kv_cache_configs(
|
||||
adjusted_memory.append(override * bytes_per_block)
|
||||
available_memory = adjusted_memory
|
||||
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
available_memory = [min(available_memory)] * len(available_memory)
|
||||
|
||||
# Reserve the null block BlockPool permanently holds back, so auto-fit and
|
||||
# the capacity check both plan against usable blocks. Allocation below
|
||||
# still uses the full memory.
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.v1.hisparse.layout import (
|
||||
HISPARSE_HOT_SUFFIX,
|
||||
HISPARSE_RESIDENT_SUFFIX,
|
||||
get_hisparse_kv_cache_groups,
|
||||
)
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
@@ -36,21 +37,20 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
|
||||
|
||||
def resolve_hisparse_block_size(
|
||||
def resolve_hisparse_specs(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
attn_layers: Mapping[str, "AttentionLayerBase"],
|
||||
) -> None:
|
||||
"""Resolve a common kernel block size in-place for HiSparse MLA specs."""
|
||||
if vllm_config.attention_config.hisparse_config is None:
|
||||
return
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
"""Resolve a common kernel block size for HiSparse MLA specs, and add the
|
||||
resident/hot specs HiSparse derives from them."""
|
||||
mla_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, MLAAttentionSpec)
|
||||
}
|
||||
if not mla_specs:
|
||||
return
|
||||
return kv_cache_spec
|
||||
block_sizes = {spec.block_size for spec in mla_specs.values()}
|
||||
if len(block_sizes) != 1:
|
||||
raise ValueError("HiSparse requires one scheduler block size.")
|
||||
@@ -62,10 +62,18 @@ def resolve_hisparse_block_size(
|
||||
"HiSparse requires a GPU block size supported by every sparse "
|
||||
f"attention and indexer backend: {error}"
|
||||
) from error
|
||||
kv_cache_spec.update(
|
||||
(name, spec.copy_with_new_block_size(block_size))
|
||||
kv_cache_spec = kv_cache_spec | {
|
||||
name: spec.copy_with_new_block_size(block_size)
|
||||
for name, spec in mla_specs.items()
|
||||
)
|
||||
}
|
||||
groups = get_hisparse_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
assert groups is not None
|
||||
return kv_cache_spec | {
|
||||
layer_name: group.kv_cache_spec
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
|
||||
for layer_name in group.layer_names
|
||||
}
|
||||
|
||||
|
||||
def allocate_hisparse_kv_caches(
|
||||
|
||||
+33
-24
@@ -47,6 +47,10 @@ class HiSparseLayout:
|
||||
def get_hisparse_kv_cache_groups(
|
||||
vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec]
|
||||
) -> list[KVCacheGroupSpec] | None:
|
||||
"""The groups HiSparse allocates: the host source group, then the GPU
|
||||
groups sharing one device pool. Derived resident/hot specs in
|
||||
`kv_cache_spec` (see `resolve_hisparse_specs`) are laid out again
|
||||
from the attention specs they belong to."""
|
||||
attention_config = getattr(vllm_config, "attention_config", None)
|
||||
if attention_config is None or attention_config.hisparse_config is None:
|
||||
return None
|
||||
@@ -59,9 +63,11 @@ def get_hisparse_kv_cache_groups(
|
||||
other_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if not isinstance(spec, MLAAttentionSpec)
|
||||
if not isinstance(
|
||||
spec, (MLAAttentionSpec, HiSparseResidentSpec, HiSparseHotSpec)
|
||||
)
|
||||
}
|
||||
if not mla_specs or not other_specs:
|
||||
if not mla_specs:
|
||||
return None
|
||||
|
||||
from vllm.v1.core.kv_cache_utils import get_kv_cache_groups
|
||||
@@ -69,7 +75,10 @@ def get_hisparse_kv_cache_groups(
|
||||
mla_group_spec = UniformTypeKVCacheSpecs.from_specs(mla_specs)
|
||||
assert mla_group_spec is not None
|
||||
mla_group = KVCacheGroupSpec(list(mla_specs), mla_group_spec)
|
||||
return [mla_group, *get_kv_cache_groups(vllm_config, other_specs)]
|
||||
regular_groups = (
|
||||
get_kv_cache_groups(vllm_config, other_specs) if other_specs else []
|
||||
)
|
||||
return _lay_out_hisparse_groups(vllm_config, [mla_group, *regular_groups])
|
||||
|
||||
|
||||
def get_hisparse_host_pool_bytes(vllm_config: VllmConfig) -> int:
|
||||
@@ -112,24 +121,9 @@ def _partition_hisparse_specs(
|
||||
return source_specs, indexer_specs
|
||||
|
||||
|
||||
def get_hisparse_gpu_memory_usage(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> int:
|
||||
_, indexer_specs = _partition_hisparse_specs(kv_cache_groups)
|
||||
return sum(
|
||||
spec.max_memory_usage_bytes(vllm_config) for spec in indexer_specs.values()
|
||||
) + sum(
|
||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config)
|
||||
for group in kv_cache_groups[1:]
|
||||
)
|
||||
|
||||
|
||||
def create_hisparse_layout(
|
||||
vllm_config: VllmConfig,
|
||||
groups: list[KVCacheGroupSpec],
|
||||
host_budget: int,
|
||||
) -> HiSparseLayout:
|
||||
def _lay_out_hisparse_groups(
|
||||
vllm_config: VllmConfig, groups: list[KVCacheGroupSpec]
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
source_specs, indexer_specs = _partition_hisparse_specs(groups)
|
||||
block_sizes = {
|
||||
spec.block_size for spec in (*source_specs.values(), *indexer_specs.values())
|
||||
@@ -225,11 +219,25 @@ def create_hisparse_layout(
|
||||
enable_kv_transfer=True,
|
||||
)
|
||||
regular_groups = groups[1:]
|
||||
gpu_groups = [indexer_group, *resident_groups, *hot_groups, *regular_groups]
|
||||
return [
|
||||
source_group,
|
||||
indexer_group,
|
||||
*resident_groups,
|
||||
*hot_groups,
|
||||
*regular_groups,
|
||||
]
|
||||
|
||||
|
||||
def create_hisparse_layout(
|
||||
vllm_config: VllmConfig,
|
||||
groups: list[KVCacheGroupSpec],
|
||||
host_budget: int,
|
||||
) -> HiSparseLayout:
|
||||
"""Size the host pool for groups laid out by `get_hisparse_kv_cache_groups`."""
|
||||
(source_group,) = [group for group in groups if group.host_resident]
|
||||
shared_host_pool = use_shared_hisparse_host_pool(vllm_config)
|
||||
host_block_stride = get_hisparse_host_block_stride(
|
||||
sum(spec.page_size_bytes for spec in source_specs.values()),
|
||||
source_group.kv_cache_spec.page_size_bytes,
|
||||
use_shared_host_pool=shared_host_pool,
|
||||
)
|
||||
host_num_blocks = host_budget // host_block_stride
|
||||
@@ -237,6 +245,7 @@ def create_hisparse_layout(
|
||||
raise ValueError("HiSparse has no allocatable host blocks.")
|
||||
# Every computed page needs a host block, so one request at max_model_len
|
||||
# must fit alongside the pool's null block and a copy-on-write tail.
|
||||
gpu_block_size = source_group.kv_cache_spec.block_size
|
||||
min_host_blocks = cdiv(vllm_config.model_config.max_model_len, gpu_block_size) + 2
|
||||
if host_num_blocks < min_host_blocks:
|
||||
raise ValueError(
|
||||
@@ -246,7 +255,7 @@ def create_hisparse_layout(
|
||||
|
||||
return HiSparseLayout(
|
||||
source_group=source_group,
|
||||
device_groups=gpu_groups,
|
||||
device_groups=[group for group in groups if not group.host_resident],
|
||||
host_num_blocks=host_num_blocks,
|
||||
host_block_stride=host_block_stride,
|
||||
shared_host_pool=shared_host_pool,
|
||||
|
||||
@@ -472,9 +472,11 @@ class HiSparseResidentSpec(KVCacheSpec):
|
||||
return cdiv(num_tokens, self.block_size)
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
return cdiv(vllm_config.model_config.max_model_len, self.block_size) * (
|
||||
self.page_size
|
||||
max_blocks = self.max_admission_blocks_per_request(
|
||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
)
|
||||
return max_blocks * self.page_size
|
||||
|
||||
@property
|
||||
def has_layer_views(self) -> bool:
|
||||
|
||||
@@ -23,7 +23,7 @@ from vllm.v1.attention.backend import (
|
||||
from vllm.v1.attention.backends.utils import create_fast_prefill_custom_backend
|
||||
from vllm.v1.hisparse.binding import (
|
||||
init_hisparse_kv_cache,
|
||||
resolve_hisparse_block_size,
|
||||
resolve_hisparse_specs,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
@@ -142,7 +142,8 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
resolve_hisparse_block_size(vllm_config, kv_cache_spec, attn_layers)
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
|
||||
return kv_cache_spec
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user