[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -661,8 +661,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
self.hash_block_size = hash_block_size
|
||||
self.dcp_world_size = dcp_world_size
|
||||
# Only groups that participate in prefix caching must satisfy the
|
||||
# divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool
|
||||
# tail, block_size=kpool) are scratch buffers and excluded.
|
||||
# divisibility constraint; groups that opt out (e.g. circular buffers
|
||||
# such as the GLM-5.3-Flash indexer tail) are scratch and excluded.
|
||||
cacheable_block_sizes = [
|
||||
manager.block_size
|
||||
for manager, group in zip(
|
||||
@@ -724,8 +724,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"""
|
||||
self.attention_groups: list[SpecGroup] = []
|
||||
for i, g in enumerate(self.kv_cache_config.kv_cache_groups):
|
||||
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail): their blocks are per-request scratch, never
|
||||
# Skip groups that opt out of prefix caching (e.g. circular
|
||||
# buffers): their blocks are per-request scratch, never
|
||||
# shareable, so they must not participate in hit lookup (their
|
||||
# manager-level hooks already no-op). Their slot in the per-group
|
||||
# hit tuple stays empty.
|
||||
|
||||
+41
-313
@@ -10,7 +10,7 @@ from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial, reduce
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, overload
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, overload
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
@@ -32,7 +32,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
HiSparseHotSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
@@ -1126,7 +1125,7 @@ def check_enough_kv_cache_memory(
|
||||
# of the specs since grouping may unify them in-place.
|
||||
groups = get_kv_cache_groups(vllm_config, dict(kv_cache_spec))
|
||||
check_memory = (
|
||||
available_memory - _pool_bytes_per_block(groups)
|
||||
available_memory - _pool_bytes_per_block(vllm_config, groups)
|
||||
if groups
|
||||
else available_memory
|
||||
)
|
||||
@@ -1237,14 +1236,17 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
|
||||
return num_blocks
|
||||
|
||||
|
||||
def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
|
||||
def _pool_bytes_per_block(
|
||||
vllm_config: VllmConfig, kv_cache_groups: list[KVCacheGroupSpec]
|
||||
) -> int:
|
||||
"""Bytes consumed by one block in the worker's shared KV cache pool, mirroring
|
||||
the divisor used by `get_kv_cache_config_from_groups` to convert
|
||||
`available_memory` into `num_blocks`. Used to compute the effective KV cache
|
||||
capacity once `num_gpu_blocks_override` is applied.
|
||||
"""
|
||||
return _get_kv_cache_bytes_per_block(
|
||||
[group for group in kv_cache_groups if not group.host_resident]
|
||||
[group for group in kv_cache_groups if not group.host_resident],
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
)
|
||||
|
||||
|
||||
@@ -1287,205 +1289,6 @@ def _get_kv_cache_groups_uniform_type(
|
||||
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
|
||||
|
||||
|
||||
def _pp_balanced_mamba_group_count(
|
||||
vllm_config: VllmConfig,
|
||||
mamba_layer_names: list[str],
|
||||
mla_layer_names: list[str],
|
||||
) -> int | None:
|
||||
"""Return a Mamba group count whose PP projections fit the MLA slots."""
|
||||
num_groups = cdiv(len(mamba_layer_names), len(mla_layer_names))
|
||||
pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
if pp_size == 1:
|
||||
return num_groups
|
||||
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
total_layers = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
mamba_indices = [extract_layer_index(name) for name in mamba_layer_names]
|
||||
mla_indices = [extract_layer_index(name) for name in mla_layer_names]
|
||||
for rank in range(pp_size):
|
||||
start, end = get_pp_indices(total_layers, rank, pp_size)
|
||||
num_mamba = sum(start <= index < end for index in mamba_indices)
|
||||
num_mla = sum(start <= index < end for index in mla_indices)
|
||||
if not num_mamba:
|
||||
continue
|
||||
if not num_mla:
|
||||
return None
|
||||
num_groups = max(num_groups, cdiv(num_mamba, num_mla))
|
||||
return num_groups
|
||||
|
||||
|
||||
def _get_kv_cache_groups_glm5_next(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec] | None:
|
||||
"""Build GLM-5.3-Flash groups with Mamba/MLA and tail/indexer aliasing."""
|
||||
mamba_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, MambaSpec)
|
||||
}
|
||||
tail_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, KpoolTailSpec)
|
||||
}
|
||||
attn_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if not isinstance(spec, (MambaSpec, KpoolTailSpec))
|
||||
}
|
||||
if not mamba_specs or not all(
|
||||
type(spec) is MLAAttentionSpec for spec in attn_specs.values()
|
||||
):
|
||||
return None
|
||||
|
||||
mla_specs = cast(dict[str, MLAAttentionSpec], attn_specs)
|
||||
idx_pages = {
|
||||
spec.page_size_bytes for spec in mla_specs.values() if spec.tokens_per_state > 1
|
||||
}
|
||||
if not idx_pages:
|
||||
return None
|
||||
|
||||
assert all(spec.page_size_padded is None for spec in mla_specs.values())
|
||||
assert len(idx_pages) == 1
|
||||
mla_names = [name for name, spec in mla_specs.items() if spec.tokens_per_state == 1]
|
||||
mla_pages = {mla_specs[name].page_size_bytes for name in mla_names}
|
||||
assert len(mla_pages) == 1
|
||||
mla_page = mla_pages.pop()
|
||||
uniform_spec = UniformTypeKVCacheSpecs.from_specs(attn_specs)
|
||||
assert uniform_spec is not None
|
||||
|
||||
tail_group: KVCacheGroupSpec | None = None
|
||||
if tail_specs:
|
||||
idx_page = next(iter(idx_pages))
|
||||
padded_tail_specs: dict[str, KVCacheSpec] = {
|
||||
name: replace(spec, page_size_padded=idx_page)
|
||||
for name, spec in tail_specs.items()
|
||||
}
|
||||
tail_uniform = UniformTypeKVCacheSpecs.from_specs(padded_tail_specs)
|
||||
assert tail_uniform is not None
|
||||
tail_group = KVCacheGroupSpec(list(padded_tail_specs), tail_uniform)
|
||||
|
||||
any_mamba = next(iter(mamba_specs.values()))
|
||||
assert all(spec == any_mamba for spec in mamba_specs.values())
|
||||
if any_mamba.real_page_size_bytes > mla_page:
|
||||
raise ValueError(
|
||||
f"the mamba state page ({any_mamba.real_page_size_bytes} bytes) "
|
||||
f"does not fit the MLA page ({mla_page} bytes); increase tensor "
|
||||
"parallelism or use a wider KV cache dtype"
|
||||
)
|
||||
padded_specs: dict[str, KVCacheSpec] = {
|
||||
name: replace(any_mamba, page_size_padded=mla_page) for name in mamba_specs
|
||||
}
|
||||
num_groups = _pp_balanced_mamba_group_count(
|
||||
vllm_config, list(mamba_specs), mla_names
|
||||
)
|
||||
if num_groups is None:
|
||||
raise ValueError(
|
||||
"a pipeline stage has mamba layers but no MLA layer to share "
|
||||
"slots with; realign the stage boundaries (VLLM_PP_LAYER_PARTITION)"
|
||||
)
|
||||
mamba_grouped_names: list[list[str]] = [[] for _ in range(num_groups)]
|
||||
for index, name in enumerate(mamba_specs):
|
||||
mamba_grouped_names[index % num_groups].append(name)
|
||||
|
||||
return (
|
||||
[KVCacheGroupSpec(list(attn_specs), uniform_spec)]
|
||||
+ ([tail_group] if tail_group is not None else [])
|
||||
+ create_kv_cache_group_specs(padded_specs, mamba_grouped_names)
|
||||
)
|
||||
|
||||
|
||||
def _glm5_next_tensor_layout(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> (
|
||||
tuple[
|
||||
KVCacheGroupSpec,
|
||||
list[KVCacheGroupSpec],
|
||||
list[str],
|
||||
list[str],
|
||||
int,
|
||||
int,
|
||||
list[str],
|
||||
int,
|
||||
]
|
||||
| None
|
||||
):
|
||||
"""Recognize the GLM-5.3-Flash grouping after optional PP projection."""
|
||||
uniform_groups = [
|
||||
group
|
||||
for group in kv_cache_groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
]
|
||||
mamba_groups = [
|
||||
group for group in kv_cache_groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
attn_group: KVCacheGroupSpec | None = None
|
||||
tail_group: KVCacheGroupSpec | None = None
|
||||
for group in uniform_groups:
|
||||
inner = cast(UniformTypeKVCacheSpecs, group.kv_cache_spec).kv_cache_specs
|
||||
if all(type(spec) is MLAAttentionSpec for spec in inner.values()):
|
||||
attn_group = group
|
||||
elif all(isinstance(spec, KpoolTailSpec) for spec in inner.values()):
|
||||
tail_group = group
|
||||
if attn_group is None or not mamba_groups:
|
||||
return None
|
||||
if len(uniform_groups) + len(mamba_groups) != len(kv_cache_groups):
|
||||
return None
|
||||
|
||||
attn_uniform = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
|
||||
mla_inner = cast(dict[str, MLAAttentionSpec], attn_uniform.kv_cache_specs)
|
||||
if not all(
|
||||
type(spec) is MLAAttentionSpec and spec.page_size_padded is None
|
||||
for spec in mla_inner.values()
|
||||
):
|
||||
return None
|
||||
mla_names = [
|
||||
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state == 1
|
||||
]
|
||||
idx_names = [
|
||||
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state > 1
|
||||
]
|
||||
mla_pages = {mla_inner[name].page_size_bytes for name in mla_names}
|
||||
idx_pages = {mla_inner[name].page_size_bytes for name in idx_names}
|
||||
if len(mla_pages) != 1 or len(idx_pages) != 1:
|
||||
return None
|
||||
mla_page = mla_pages.pop()
|
||||
idx_page = idx_pages.pop()
|
||||
if any(group.kv_cache_spec.page_size_bytes != mla_page for group in mamba_groups):
|
||||
return None
|
||||
|
||||
tail_names: list[str] = []
|
||||
tail_page = 0
|
||||
if tail_group is not None:
|
||||
tail_names = list(tail_group.layer_names)
|
||||
tail_inner = cast(
|
||||
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
tail_pages = {
|
||||
cast(KpoolTailSpec, spec).unpadded_page_size_bytes
|
||||
for spec in tail_inner.values()
|
||||
}
|
||||
if len(tail_pages) != 1 or len(tail_names) != len(idx_names):
|
||||
return None
|
||||
tail_page = tail_pages.pop()
|
||||
if tail_page > idx_page:
|
||||
return None
|
||||
|
||||
return (
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
tail_page,
|
||||
)
|
||||
|
||||
|
||||
def unify_kv_cache_spec_page_size(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
@@ -1709,14 +1512,24 @@ def _get_per_layer_spec(
|
||||
return spec
|
||||
|
||||
|
||||
def _get_layer_block_stride(spec: KVCacheSpec) -> int:
|
||||
"""A layer-outer block stride: the page padded to its alignment."""
|
||||
return round_up(spec.page_size_bytes, spec.get_block_stride_alignment())
|
||||
|
||||
|
||||
def _get_kv_cache_bytes_per_block(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
kv_cache_groups: list[KVCacheGroupSpec], layout: KVCacheLayout
|
||||
) -> int:
|
||||
"""Return the largest cache group's bytes per block."""
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
_, _, mla_names, idx_names, mla_page, idx_page, _, _ = glm5_layout
|
||||
return len(mla_names) * mla_page + len(idx_names) * idx_page
|
||||
|
||||
if not layout.is_block_outermost:
|
||||
# Each layer's block stride is its own padded page.
|
||||
return max(
|
||||
sum(
|
||||
_get_layer_block_stride(_get_per_layer_spec(group, layer_name))
|
||||
for layer_name in group.layer_names
|
||||
)
|
||||
for group in kv_cache_groups
|
||||
)
|
||||
bytes_per_block = max(
|
||||
sum(
|
||||
_get_per_layer_spec(group, layer_name).page_size_bytes
|
||||
@@ -1731,7 +1544,7 @@ def _get_kv_cache_bytes_per_block(
|
||||
if isinstance(group.kv_cache_spec, HiSparseHotSpec)
|
||||
]
|
||||
alignments.extend(
|
||||
_get_per_layer_spec(group, layer_name).block_stride_alignment or 1
|
||||
_get_per_layer_spec(group, layer_name).get_block_stride_alignment()
|
||||
for group in kv_cache_groups
|
||||
for layer_name in group.layer_names
|
||||
)
|
||||
@@ -1806,72 +1619,9 @@ def get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, available_memory, host_budget
|
||||
)
|
||||
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
(
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
_,
|
||||
) = glm5_layout
|
||||
bytes_per_block = len(mla_names) * mla_page + len(idx_names) * idx_page
|
||||
num_blocks = may_override_num_blocks(
|
||||
vllm_config, available_memory // bytes_per_block
|
||||
)
|
||||
size = bytes_per_block * num_blocks
|
||||
attn_specs = cast(
|
||||
UniformTypeKVCacheSpecs, attn_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
|
||||
kv_cache_tensors: list[KVCacheTensor] = []
|
||||
|
||||
def add_tensor(layer_name: str, spec: KVCacheSpec, offset: int) -> None:
|
||||
kv_cache_tensors.append(
|
||||
KVCacheTensor(
|
||||
size=size,
|
||||
layers=[layer_name],
|
||||
layer_stride=spec.page_size_bytes * num_blocks,
|
||||
block_stride=spec.page_size_bytes,
|
||||
offset=offset,
|
||||
)
|
||||
)
|
||||
|
||||
for index, mla_name in enumerate(mla_names):
|
||||
offset = index * mla_page * num_blocks
|
||||
add_tensor(mla_name, attn_specs[mla_name], offset)
|
||||
for group in mamba_groups:
|
||||
if index < len(group.layer_names):
|
||||
add_tensor(group.layer_names[index], group.kv_cache_spec, offset)
|
||||
|
||||
idx_base = len(mla_names) * mla_page * num_blocks
|
||||
for index, idx_name in enumerate(idx_names):
|
||||
offset = idx_base + index * idx_page * num_blocks
|
||||
add_tensor(idx_name, attn_specs[idx_name], offset)
|
||||
if tail_names:
|
||||
tail_name = tail_names[index]
|
||||
tail_group = next(
|
||||
group for group in kv_cache_groups if tail_name in group.layer_names
|
||||
)
|
||||
tail_specs = cast(
|
||||
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
add_tensor(tail_name, tail_specs[tail_name], offset)
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
validate_kv_cache_layout(layout, kv_cache_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups, layout)
|
||||
interleaved_block_stride = bytes_per_block if layout.is_block_outermost else None
|
||||
|
||||
num_blocks = available_memory // bytes_per_block
|
||||
@@ -1901,17 +1651,17 @@ def get_kv_cache_config_from_groups(
|
||||
|
||||
byte_offset = 0
|
||||
for spec, layer_names in layers_by_spec.items():
|
||||
block_stride = interleaved_block_stride
|
||||
padded_page = _get_layer_block_stride(spec)
|
||||
if block_stride is None and padded_page != spec.page_size_bytes:
|
||||
assert layout.is_block_compact, f"{layout.name} cannot pad pages"
|
||||
block_stride = padded_page
|
||||
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
||||
spec,
|
||||
num_blocks,
|
||||
len(layer_names),
|
||||
layout,
|
||||
fixed_strides=(None, interleaved_block_stride, None, None, None),
|
||||
)
|
||||
offset = (
|
||||
byte_offset
|
||||
* max(layer_stride, spec.page_size_bytes)
|
||||
// spec.page_size_bytes
|
||||
fixed_strides=(None, block_stride, None, None, None),
|
||||
)
|
||||
kv_cache_tensors.append(
|
||||
KVCacheTensor(
|
||||
@@ -1919,10 +1669,13 @@ def get_kv_cache_config_from_groups(
|
||||
layers=layer_names,
|
||||
layer_stride=layer_stride,
|
||||
block_stride=block_stride,
|
||||
offset=offset,
|
||||
offset=byte_offset,
|
||||
)
|
||||
)
|
||||
byte_offset += len(layer_names) * spec.page_size_bytes
|
||||
# Block-outer layers share each block; layer-outer ones own regions.
|
||||
byte_offset += len(layer_names) * (
|
||||
spec.page_size_bytes if layout.is_block_outermost else layer_stride
|
||||
)
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
@@ -2438,9 +2191,6 @@ def get_kv_cache_groups(
|
||||
# full attention, or all layers are sliding window attention with the
|
||||
# same window size). Put all layers into one group.
|
||||
return _get_kv_cache_groups_uniform_type(uniform_spec)
|
||||
elif glm5_groups := _get_kv_cache_groups_glm5_next(vllm_config, kv_cache_spec):
|
||||
return glm5_groups
|
||||
|
||||
# Hidden-state layers use their own block table and must not be absorbed
|
||||
# into a compatible attention bucket.
|
||||
hidden_specs = {
|
||||
@@ -2594,31 +2344,7 @@ def _max_memory_usage_bytes_from_groups(
|
||||
if not kv_cache_groups:
|
||||
return 0
|
||||
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
(
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
_,
|
||||
) = glm5_layout
|
||||
uniform_spec = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
|
||||
total_blocks = uniform_spec.max_memory_usage_pages(vllm_config)
|
||||
total_blocks += sum(
|
||||
cdiv(
|
||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
||||
group.kv_cache_spec.page_size_bytes,
|
||||
)
|
||||
for group in mamba_groups
|
||||
)
|
||||
if tail_names:
|
||||
total_blocks += 1
|
||||
return total_blocks * (len(mla_names) * mla_page + len(idx_names) * idx_page)
|
||||
|
||||
bytes_per_block = _pool_bytes_per_block(kv_cache_groups)
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
total_blocks = 0
|
||||
for group in kv_cache_groups:
|
||||
spec = group.kv_cache_spec
|
||||
@@ -2874,7 +2600,7 @@ def get_kv_cache_configs(
|
||||
if not groups:
|
||||
adjusted_memory.append(avail_mem)
|
||||
continue
|
||||
bytes_per_block = _pool_bytes_per_block(groups)
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, groups)
|
||||
logger.info(
|
||||
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
|
||||
avail_mem // bytes_per_block,
|
||||
@@ -2887,7 +2613,7 @@ def get_kv_cache_configs(
|
||||
# the capacity check both plan against usable blocks. Allocation below
|
||||
# still uses the full memory.
|
||||
check_memory = [
|
||||
avail_mem - _pool_bytes_per_block(groups) if groups else avail_mem
|
||||
avail_mem - _pool_bytes_per_block(vllm_config, groups) if groups else avail_mem
|
||||
for groups, avail_mem in zip(projected_groups_per_worker, available_memory)
|
||||
]
|
||||
|
||||
@@ -2931,7 +2657,9 @@ def get_kv_cache_configs(
|
||||
# strides and offsets stay consistent with the shrunken allocation.
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
kv_cache_configs[i] = get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, min_num_blocks * _pool_bytes_per_block(groups)
|
||||
vllm_config,
|
||||
groups,
|
||||
min_num_blocks * _pool_bytes_per_block(vllm_config, groups),
|
||||
)
|
||||
|
||||
for kv_cache_config in kv_cache_configs:
|
||||
|
||||
@@ -7,7 +7,6 @@ from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
@@ -27,7 +26,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
HiddenStateCacheSpec,
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheGroupRole,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
@@ -45,8 +43,6 @@ from vllm.v1.request import Request
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SingleTypeKVCacheManager(ABC):
|
||||
"""An abstract base class for a manager that handle the kv cache management
|
||||
@@ -1299,10 +1295,6 @@ class CircularBufferManager(FullAttentionManager):
|
||||
return 0
|
||||
|
||||
|
||||
class KpoolTailManager(CircularBufferManager):
|
||||
"""One-block circular scratch manager for ``KpoolTailSpec``."""
|
||||
|
||||
|
||||
class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec, **kwargs) -> None:
|
||||
super().__init__(kv_cache_spec, **kwargs)
|
||||
@@ -2687,11 +2679,6 @@ def register_all_kvcache_specs(vllm_config):
|
||||
SlidingWindowManager,
|
||||
uniform_type_base_spec=SlidingWindowMLASpec,
|
||||
)
|
||||
KVCacheSpecRegistry.register(
|
||||
KpoolTailSpec,
|
||||
KpoolTailManager,
|
||||
uniform_type_base_spec=KpoolTailSpec,
|
||||
)
|
||||
|
||||
KVCacheSpecRegistry.register(
|
||||
MambaSpec, MambaManager, uniform_type_base_spec=MambaSpec
|
||||
|
||||
Reference in New Issue
Block a user