[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -599,6 +599,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
# can be a multiple of hash_block_size.
|
||||
self.hash_block_size = hash_block_size
|
||||
self.dcp_world_size = dcp_world_size
|
||||
# Only groups that participate in prefix caching must satisfy the
|
||||
# divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool
|
||||
# tail, block_size=kpool) are scratch buffers and excluded.
|
||||
group_block_sizes = [
|
||||
manager.block_size
|
||||
for manager, group in zip(
|
||||
@@ -679,6 +682,11 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"""
|
||||
self.attention_groups: list[SpecGroup] = []
|
||||
for i, g in enumerate(self.kv_cache_config.kv_cache_groups):
|
||||
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail): their blocks are per-request scratch, never
|
||||
# shareable, so they must not participate in hit lookup (their
|
||||
# manager-level hooks already no-op). Their slot in the per-group
|
||||
# hit tuple stays empty.
|
||||
if not g.kv_cache_spec.prefix_cacheable:
|
||||
continue
|
||||
manager_cls = self.single_type_managers[i].__class__
|
||||
|
||||
@@ -10,7 +10,7 @@ from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import partial
|
||||
from typing import Any, NamedTuple, NewType, TypeAlias, overload
|
||||
from typing import Any, NamedTuple, NewType, TypeAlias, cast, overload
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
@@ -24,6 +24,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
@@ -1132,6 +1133,205 @@ def _get_kv_cache_groups_uniform_type(
|
||||
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
|
||||
|
||||
|
||||
def _pp_balanced_mamba_group_count(
|
||||
vllm_config: VllmConfig,
|
||||
mamba_layer_names: list[str],
|
||||
mla_layer_names: list[str],
|
||||
) -> int | None:
|
||||
"""Return a Mamba group count whose PP projections fit the MLA slots."""
|
||||
num_groups = cdiv(len(mamba_layer_names), len(mla_layer_names))
|
||||
pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
if pp_size == 1:
|
||||
return num_groups
|
||||
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
total_layers = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
mamba_indices = [extract_layer_index(name) for name in mamba_layer_names]
|
||||
mla_indices = [extract_layer_index(name) for name in mla_layer_names]
|
||||
for rank in range(pp_size):
|
||||
start, end = get_pp_indices(total_layers, rank, pp_size)
|
||||
num_mamba = sum(start <= index < end for index in mamba_indices)
|
||||
num_mla = sum(start <= index < end for index in mla_indices)
|
||||
if not num_mamba:
|
||||
continue
|
||||
if not num_mla:
|
||||
return None
|
||||
num_groups = max(num_groups, cdiv(num_mamba, num_mla))
|
||||
return num_groups
|
||||
|
||||
|
||||
def _get_kv_cache_groups_glm5_next(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec] | None:
|
||||
"""Build GLM-5.3-Flash groups with Mamba/MLA and tail/indexer aliasing."""
|
||||
mamba_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, MambaSpec)
|
||||
}
|
||||
tail_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, KpoolTailSpec)
|
||||
}
|
||||
attn_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if not isinstance(spec, (MambaSpec, KpoolTailSpec))
|
||||
}
|
||||
if not mamba_specs or not all(
|
||||
type(spec) is MLAAttentionSpec for spec in attn_specs.values()
|
||||
):
|
||||
return None
|
||||
|
||||
mla_specs = cast(dict[str, MLAAttentionSpec], attn_specs)
|
||||
idx_pages = {
|
||||
spec.page_size_bytes for spec in mla_specs.values() if spec.tokens_per_state > 1
|
||||
}
|
||||
if not idx_pages:
|
||||
return None
|
||||
|
||||
assert all(spec.page_size_padded is None for spec in mla_specs.values())
|
||||
assert len(idx_pages) == 1
|
||||
mla_names = [name for name, spec in mla_specs.items() if spec.tokens_per_state == 1]
|
||||
mla_pages = {mla_specs[name].page_size_bytes for name in mla_names}
|
||||
assert len(mla_pages) == 1
|
||||
mla_page = mla_pages.pop()
|
||||
uniform_spec = UniformTypeKVCacheSpecs.from_specs(attn_specs)
|
||||
assert uniform_spec is not None
|
||||
|
||||
tail_group: KVCacheGroupSpec | None = None
|
||||
if tail_specs:
|
||||
idx_page = next(iter(idx_pages))
|
||||
padded_tail_specs: dict[str, KVCacheSpec] = {
|
||||
name: replace(spec, page_size_padded=idx_page)
|
||||
for name, spec in tail_specs.items()
|
||||
}
|
||||
tail_uniform = UniformTypeKVCacheSpecs.from_specs(padded_tail_specs)
|
||||
assert tail_uniform is not None
|
||||
tail_group = KVCacheGroupSpec(list(padded_tail_specs), tail_uniform)
|
||||
|
||||
any_mamba = next(iter(mamba_specs.values()))
|
||||
assert all(spec == any_mamba for spec in mamba_specs.values())
|
||||
if any_mamba.real_page_size_bytes > mla_page:
|
||||
raise ValueError(
|
||||
f"the mamba state page ({any_mamba.real_page_size_bytes} bytes) "
|
||||
f"does not fit the MLA page ({mla_page} bytes); increase tensor "
|
||||
"parallelism or use a wider KV cache dtype"
|
||||
)
|
||||
padded_specs: dict[str, KVCacheSpec] = {
|
||||
name: replace(any_mamba, page_size_padded=mla_page) for name in mamba_specs
|
||||
}
|
||||
num_groups = _pp_balanced_mamba_group_count(
|
||||
vllm_config, list(mamba_specs), mla_names
|
||||
)
|
||||
if num_groups is None:
|
||||
raise ValueError(
|
||||
"a pipeline stage has mamba layers but no MLA layer to share "
|
||||
"slots with; realign the stage boundaries (VLLM_PP_LAYER_PARTITION)"
|
||||
)
|
||||
mamba_grouped_names: list[list[str]] = [[] for _ in range(num_groups)]
|
||||
for index, name in enumerate(mamba_specs):
|
||||
mamba_grouped_names[index % num_groups].append(name)
|
||||
|
||||
return (
|
||||
[KVCacheGroupSpec(list(attn_specs), uniform_spec)]
|
||||
+ ([tail_group] if tail_group is not None else [])
|
||||
+ create_kv_cache_group_specs(padded_specs, mamba_grouped_names)
|
||||
)
|
||||
|
||||
|
||||
def _glm5_next_tensor_layout(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> (
|
||||
tuple[
|
||||
KVCacheGroupSpec,
|
||||
list[KVCacheGroupSpec],
|
||||
list[str],
|
||||
list[str],
|
||||
int,
|
||||
int,
|
||||
list[str],
|
||||
int,
|
||||
]
|
||||
| None
|
||||
):
|
||||
"""Recognize the GLM-5.3-Flash grouping after optional PP projection."""
|
||||
uniform_groups = [
|
||||
group
|
||||
for group in kv_cache_groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
]
|
||||
mamba_groups = [
|
||||
group for group in kv_cache_groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
attn_group: KVCacheGroupSpec | None = None
|
||||
tail_group: KVCacheGroupSpec | None = None
|
||||
for group in uniform_groups:
|
||||
inner = cast(UniformTypeKVCacheSpecs, group.kv_cache_spec).kv_cache_specs
|
||||
if all(type(spec) is MLAAttentionSpec for spec in inner.values()):
|
||||
attn_group = group
|
||||
elif all(isinstance(spec, KpoolTailSpec) for spec in inner.values()):
|
||||
tail_group = group
|
||||
if attn_group is None or not mamba_groups:
|
||||
return None
|
||||
if len(uniform_groups) + len(mamba_groups) != len(kv_cache_groups):
|
||||
return None
|
||||
|
||||
attn_uniform = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
|
||||
mla_inner = cast(dict[str, MLAAttentionSpec], attn_uniform.kv_cache_specs)
|
||||
if not all(
|
||||
type(spec) is MLAAttentionSpec and spec.page_size_padded is None
|
||||
for spec in mla_inner.values()
|
||||
):
|
||||
return None
|
||||
mla_names = [
|
||||
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state == 1
|
||||
]
|
||||
idx_names = [
|
||||
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state > 1
|
||||
]
|
||||
mla_pages = {mla_inner[name].page_size_bytes for name in mla_names}
|
||||
idx_pages = {mla_inner[name].page_size_bytes for name in idx_names}
|
||||
if len(mla_pages) != 1 or len(idx_pages) != 1:
|
||||
return None
|
||||
mla_page = mla_pages.pop()
|
||||
idx_page = idx_pages.pop()
|
||||
if any(group.kv_cache_spec.page_size_bytes != mla_page for group in mamba_groups):
|
||||
return None
|
||||
|
||||
tail_names: list[str] = []
|
||||
tail_page = 0
|
||||
if tail_group is not None:
|
||||
tail_names = list(tail_group.layer_names)
|
||||
tail_inner = cast(
|
||||
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
tail_pages = {
|
||||
cast(KpoolTailSpec, spec).unpadded_page_size_bytes
|
||||
for spec in tail_inner.values()
|
||||
}
|
||||
if len(tail_pages) != 1 or len(tail_names) != len(idx_names):
|
||||
return None
|
||||
tail_page = tail_pages.pop()
|
||||
if tail_page > idx_page:
|
||||
return None
|
||||
|
||||
return (
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
tail_page,
|
||||
)
|
||||
|
||||
|
||||
def unify_kv_cache_spec_page_size(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
@@ -1359,6 +1559,10 @@ def _get_kv_cache_bytes_per_block(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> int:
|
||||
"""Return the largest cache group's bytes per block."""
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
_, _, mla_names, idx_names, mla_page, idx_page, _, _ = glm5_layout
|
||||
return len(mla_names) * mla_page + len(idx_names) * idx_page
|
||||
|
||||
bytes_per_block = max(
|
||||
sum(
|
||||
_get_per_layer_spec(group, layer_name).page_size_bytes
|
||||
@@ -1432,6 +1636,69 @@ def get_kv_cache_config_from_groups(
|
||||
),
|
||||
)
|
||||
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
(
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
_,
|
||||
) = glm5_layout
|
||||
bytes_per_block = len(mla_names) * mla_page + len(idx_names) * idx_page
|
||||
num_blocks = may_override_num_blocks(
|
||||
vllm_config, available_memory // bytes_per_block
|
||||
)
|
||||
size = bytes_per_block * num_blocks
|
||||
attn_specs = cast(
|
||||
UniformTypeKVCacheSpecs, attn_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
|
||||
kv_cache_tensors: list[KVCacheTensor] = []
|
||||
|
||||
def add_tensor(layer_name: str, spec: KVCacheSpec, offset: int) -> None:
|
||||
kv_cache_tensors.append(
|
||||
KVCacheTensor(
|
||||
size=size,
|
||||
layers=[layer_name],
|
||||
layer_stride=spec.page_size_bytes * num_blocks,
|
||||
block_stride=spec.page_size_bytes,
|
||||
offset=offset,
|
||||
)
|
||||
)
|
||||
|
||||
for index, mla_name in enumerate(mla_names):
|
||||
offset = index * mla_page * num_blocks
|
||||
add_tensor(mla_name, attn_specs[mla_name], offset)
|
||||
for group in mamba_groups:
|
||||
if index < len(group.layer_names):
|
||||
add_tensor(group.layer_names[index], group.kv_cache_spec, offset)
|
||||
|
||||
idx_base = len(mla_names) * mla_page * num_blocks
|
||||
for index, idx_name in enumerate(idx_names):
|
||||
offset = idx_base + index * idx_page * num_blocks
|
||||
add_tensor(idx_name, attn_specs[idx_name], offset)
|
||||
if tail_names:
|
||||
tail_name = tail_names[index]
|
||||
tail_group = next(
|
||||
group for group in kv_cache_groups if tail_name in group.layer_names
|
||||
)
|
||||
tail_specs = cast(
|
||||
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
add_tensor(tail_name, tail_specs[tail_name], offset)
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
validate_kv_cache_layout(layout, kv_cache_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups)
|
||||
@@ -1941,6 +2208,9 @@ def get_kv_cache_groups(
|
||||
# full attention, or all layers are sliding window attention with the
|
||||
# same window size). Put all layers into one group.
|
||||
return _get_kv_cache_groups_uniform_type(uniform_spec)
|
||||
elif glm5_groups := _get_kv_cache_groups_glm5_next(vllm_config, kv_cache_spec):
|
||||
return glm5_groups
|
||||
|
||||
# Hidden-state layers use their own block table and must not be absorbed
|
||||
# into a compatible attention bucket.
|
||||
hidden_specs = {
|
||||
@@ -2065,6 +2335,30 @@ def _max_memory_usage_bytes_from_groups(
|
||||
if not kv_cache_groups:
|
||||
return 0
|
||||
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
(
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
_,
|
||||
) = glm5_layout
|
||||
uniform_spec = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
|
||||
total_blocks = uniform_spec.max_memory_usage_pages(vllm_config)
|
||||
total_blocks += sum(
|
||||
cdiv(
|
||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
||||
group.kv_cache_spec.page_size_bytes,
|
||||
)
|
||||
for group in mamba_groups
|
||||
)
|
||||
if tail_names:
|
||||
total_blocks += 1
|
||||
return total_blocks * (len(mla_names) * mla_page + len(idx_names) * idx_page)
|
||||
|
||||
bytes_per_block = _pool_bytes_per_block(kv_cache_groups)
|
||||
total_blocks = 0
|
||||
for group in kv_cache_groups:
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections import defaultdict
|
||||
from collections.abc import Sequence
|
||||
from typing import ClassVar
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
@@ -22,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
CrossAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
@@ -33,6 +35,8 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SingleTypeKVCacheManager(ABC):
|
||||
"""
|
||||
@@ -1210,6 +1214,10 @@ class CircularBufferManager(FullAttentionManager):
|
||||
return 0
|
||||
|
||||
|
||||
class KpoolTailManager(CircularBufferManager):
|
||||
"""One-block circular scratch manager for ``KpoolTailSpec``."""
|
||||
|
||||
|
||||
class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec, **kwargs) -> None:
|
||||
super().__init__(kv_cache_spec, **kwargs)
|
||||
@@ -2085,6 +2093,11 @@ def register_all_kvcache_specs(vllm_config):
|
||||
SlidingWindowManager,
|
||||
uniform_type_base_spec=SlidingWindowMLASpec,
|
||||
)
|
||||
KVCacheSpecRegistry.register(
|
||||
KpoolTailSpec,
|
||||
KpoolTailManager,
|
||||
uniform_type_base_spec=KpoolTailSpec,
|
||||
)
|
||||
|
||||
KVCacheSpecRegistry.register(
|
||||
MambaSpec, MambaManager, uniform_type_base_spec=MambaSpec
|
||||
|
||||
Reference in New Issue
Block a user