[Model] add GLM-5.3-Flash support (#53906)

This commit is contained in:
Jiangyun Zhu
2026-09-03 09:40:35 -07:00
committed by GitHub
parent da8ec28268
commit 98ed0856f3
94 changed files with 17927 additions and 266 deletions
+8
View File
@@ -599,6 +599,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
# can be a multiple of hash_block_size.
self.hash_block_size = hash_block_size
self.dcp_world_size = dcp_world_size
# Only groups that participate in prefix caching must satisfy the
# divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool
# tail, block_size=kpool) are scratch buffers and excluded.
group_block_sizes = [
manager.block_size
for manager, group in zip(
@@ -679,6 +682,11 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
"""
self.attention_groups: list[SpecGroup] = []
for i, g in enumerate(self.kv_cache_config.kv_cache_groups):
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
# kpool tail): their blocks are per-request scratch, never
# shareable, so they must not participate in hit lookup (their
# manager-level hooks already no-op). Their slot in the per-group
# hit tuple stays empty.
if not g.kv_cache_spec.prefix_cacheable:
continue
manager_cls = self.single_type_managers[i].__class__
+295 -1
View File
@@ -10,7 +10,7 @@ from collections import defaultdict
from collections.abc import Callable, Iterable, Iterator, Sequence
from dataclasses import dataclass, replace
from functools import partial
from typing import Any, NamedTuple, NewType, TypeAlias, overload
from typing import Any, NamedTuple, NewType, TypeAlias, cast, overload
from vllm import envs
from vllm.config import VllmConfig
@@ -24,6 +24,7 @@ from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
@@ -1132,6 +1133,205 @@ def _get_kv_cache_groups_uniform_type(
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
def _pp_balanced_mamba_group_count(
vllm_config: VllmConfig,
mamba_layer_names: list[str],
mla_layer_names: list[str],
) -> int | None:
"""Return a Mamba group count whose PP projections fit the MLA slots."""
num_groups = cdiv(len(mamba_layer_names), len(mla_layer_names))
pp_size = vllm_config.parallel_config.pipeline_parallel_size
if pp_size == 1:
return num_groups
from vllm.distributed.utils import get_pp_indices
from vllm.model_executor.models.utils import extract_layer_index
total_layers = vllm_config.model_config.get_total_num_hidden_layers()
mamba_indices = [extract_layer_index(name) for name in mamba_layer_names]
mla_indices = [extract_layer_index(name) for name in mla_layer_names]
for rank in range(pp_size):
start, end = get_pp_indices(total_layers, rank, pp_size)
num_mamba = sum(start <= index < end for index in mamba_indices)
num_mla = sum(start <= index < end for index in mla_indices)
if not num_mamba:
continue
if not num_mla:
return None
num_groups = max(num_groups, cdiv(num_mamba, num_mla))
return num_groups
def _get_kv_cache_groups_glm5_next(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec] | None:
"""Build GLM-5.3-Flash groups with Mamba/MLA and tail/indexer aliasing."""
mamba_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if isinstance(spec, MambaSpec)
}
tail_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if isinstance(spec, KpoolTailSpec)
}
attn_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if not isinstance(spec, (MambaSpec, KpoolTailSpec))
}
if not mamba_specs or not all(
type(spec) is MLAAttentionSpec for spec in attn_specs.values()
):
return None
mla_specs = cast(dict[str, MLAAttentionSpec], attn_specs)
idx_pages = {
spec.page_size_bytes for spec in mla_specs.values() if spec.tokens_per_state > 1
}
if not idx_pages:
return None
assert all(spec.page_size_padded is None for spec in mla_specs.values())
assert len(idx_pages) == 1
mla_names = [name for name, spec in mla_specs.items() if spec.tokens_per_state == 1]
mla_pages = {mla_specs[name].page_size_bytes for name in mla_names}
assert len(mla_pages) == 1
mla_page = mla_pages.pop()
uniform_spec = UniformTypeKVCacheSpecs.from_specs(attn_specs)
assert uniform_spec is not None
tail_group: KVCacheGroupSpec | None = None
if tail_specs:
idx_page = next(iter(idx_pages))
padded_tail_specs: dict[str, KVCacheSpec] = {
name: replace(spec, page_size_padded=idx_page)
for name, spec in tail_specs.items()
}
tail_uniform = UniformTypeKVCacheSpecs.from_specs(padded_tail_specs)
assert tail_uniform is not None
tail_group = KVCacheGroupSpec(list(padded_tail_specs), tail_uniform)
any_mamba = next(iter(mamba_specs.values()))
assert all(spec == any_mamba for spec in mamba_specs.values())
if any_mamba.real_page_size_bytes > mla_page:
raise ValueError(
f"the mamba state page ({any_mamba.real_page_size_bytes} bytes) "
f"does not fit the MLA page ({mla_page} bytes); increase tensor "
"parallelism or use a wider KV cache dtype"
)
padded_specs: dict[str, KVCacheSpec] = {
name: replace(any_mamba, page_size_padded=mla_page) for name in mamba_specs
}
num_groups = _pp_balanced_mamba_group_count(
vllm_config, list(mamba_specs), mla_names
)
if num_groups is None:
raise ValueError(
"a pipeline stage has mamba layers but no MLA layer to share "
"slots with; realign the stage boundaries (VLLM_PP_LAYER_PARTITION)"
)
mamba_grouped_names: list[list[str]] = [[] for _ in range(num_groups)]
for index, name in enumerate(mamba_specs):
mamba_grouped_names[index % num_groups].append(name)
return (
[KVCacheGroupSpec(list(attn_specs), uniform_spec)]
+ ([tail_group] if tail_group is not None else [])
+ create_kv_cache_group_specs(padded_specs, mamba_grouped_names)
)
def _glm5_next_tensor_layout(
kv_cache_groups: list[KVCacheGroupSpec],
) -> (
tuple[
KVCacheGroupSpec,
list[KVCacheGroupSpec],
list[str],
list[str],
int,
int,
list[str],
int,
]
| None
):
"""Recognize the GLM-5.3-Flash grouping after optional PP projection."""
uniform_groups = [
group
for group in kv_cache_groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
]
mamba_groups = [
group for group in kv_cache_groups if isinstance(group.kv_cache_spec, MambaSpec)
]
attn_group: KVCacheGroupSpec | None = None
tail_group: KVCacheGroupSpec | None = None
for group in uniform_groups:
inner = cast(UniformTypeKVCacheSpecs, group.kv_cache_spec).kv_cache_specs
if all(type(spec) is MLAAttentionSpec for spec in inner.values()):
attn_group = group
elif all(isinstance(spec, KpoolTailSpec) for spec in inner.values()):
tail_group = group
if attn_group is None or not mamba_groups:
return None
if len(uniform_groups) + len(mamba_groups) != len(kv_cache_groups):
return None
attn_uniform = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
mla_inner = cast(dict[str, MLAAttentionSpec], attn_uniform.kv_cache_specs)
if not all(
type(spec) is MLAAttentionSpec and spec.page_size_padded is None
for spec in mla_inner.values()
):
return None
mla_names = [
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state == 1
]
idx_names = [
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state > 1
]
mla_pages = {mla_inner[name].page_size_bytes for name in mla_names}
idx_pages = {mla_inner[name].page_size_bytes for name in idx_names}
if len(mla_pages) != 1 or len(idx_pages) != 1:
return None
mla_page = mla_pages.pop()
idx_page = idx_pages.pop()
if any(group.kv_cache_spec.page_size_bytes != mla_page for group in mamba_groups):
return None
tail_names: list[str] = []
tail_page = 0
if tail_group is not None:
tail_names = list(tail_group.layer_names)
tail_inner = cast(
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
).kv_cache_specs
tail_pages = {
cast(KpoolTailSpec, spec).unpadded_page_size_bytes
for spec in tail_inner.values()
}
if len(tail_pages) != 1 or len(tail_names) != len(idx_names):
return None
tail_page = tail_pages.pop()
if tail_page > idx_page:
return None
return (
attn_group,
mamba_groups,
mla_names,
idx_names,
mla_page,
idx_page,
tail_names,
tail_page,
)
def unify_kv_cache_spec_page_size(
kv_cache_spec: dict[str, KVCacheSpec],
) -> dict[str, KVCacheSpec]:
@@ -1359,6 +1559,10 @@ def _get_kv_cache_bytes_per_block(
kv_cache_groups: list[KVCacheGroupSpec],
) -> int:
"""Return the largest cache group's bytes per block."""
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
_, _, mla_names, idx_names, mla_page, idx_page, _, _ = glm5_layout
return len(mla_names) * mla_page + len(idx_names) * idx_page
bytes_per_block = max(
sum(
_get_per_layer_spec(group, layer_name).page_size_bytes
@@ -1432,6 +1636,69 @@ def get_kv_cache_config_from_groups(
),
)
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
(
attn_group,
mamba_groups,
mla_names,
idx_names,
mla_page,
idx_page,
tail_names,
_,
) = glm5_layout
bytes_per_block = len(mla_names) * mla_page + len(idx_names) * idx_page
num_blocks = may_override_num_blocks(
vllm_config, available_memory // bytes_per_block
)
size = bytes_per_block * num_blocks
attn_specs = cast(
UniformTypeKVCacheSpecs, attn_group.kv_cache_spec
).kv_cache_specs
kv_cache_tensors: list[KVCacheTensor] = []
def add_tensor(layer_name: str, spec: KVCacheSpec, offset: int) -> None:
kv_cache_tensors.append(
KVCacheTensor(
size=size,
layers=[layer_name],
layer_stride=spec.page_size_bytes * num_blocks,
block_stride=spec.page_size_bytes,
offset=offset,
)
)
for index, mla_name in enumerate(mla_names):
offset = index * mla_page * num_blocks
add_tensor(mla_name, attn_specs[mla_name], offset)
for group in mamba_groups:
if index < len(group.layer_names):
add_tensor(group.layer_names[index], group.kv_cache_spec, offset)
idx_base = len(mla_names) * mla_page * num_blocks
for index, idx_name in enumerate(idx_names):
offset = idx_base + index * idx_page * num_blocks
add_tensor(idx_name, attn_specs[idx_name], offset)
if tail_names:
tail_name = tail_names[index]
tail_group = next(
group for group in kv_cache_groups if tail_name in group.layer_names
)
tail_specs = cast(
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
).kv_cache_specs
add_tensor(tail_name, tail_specs[tail_name], offset)
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=kv_cache_groups,
prefix_cache_retention_interval=(
vllm_config.cache_config.prefix_cache_retention_interval
),
)
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
validate_kv_cache_layout(layout, kv_cache_groups)
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups)
@@ -1941,6 +2208,9 @@ def get_kv_cache_groups(
# full attention, or all layers are sliding window attention with the
# same window size). Put all layers into one group.
return _get_kv_cache_groups_uniform_type(uniform_spec)
elif glm5_groups := _get_kv_cache_groups_glm5_next(vllm_config, kv_cache_spec):
return glm5_groups
# Hidden-state layers use their own block table and must not be absorbed
# into a compatible attention bucket.
hidden_specs = {
@@ -2065,6 +2335,30 @@ def _max_memory_usage_bytes_from_groups(
if not kv_cache_groups:
return 0
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
(
attn_group,
mamba_groups,
mla_names,
idx_names,
mla_page,
idx_page,
tail_names,
_,
) = glm5_layout
uniform_spec = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
total_blocks = uniform_spec.max_memory_usage_pages(vllm_config)
total_blocks += sum(
cdiv(
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
group.kv_cache_spec.page_size_bytes,
)
for group in mamba_groups
)
if tail_names:
total_blocks += 1
return total_blocks * (len(mla_names) * mla_page + len(idx_names) * idx_page)
bytes_per_block = _pool_bytes_per_block(kv_cache_groups)
total_blocks = 0
for group in kv_cache_groups:
@@ -6,6 +6,7 @@ from collections import defaultdict
from collections.abc import Sequence
from typing import ClassVar
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import (
@@ -22,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
CrossAttentionSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
KpoolTailSpec,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
@@ -33,6 +35,8 @@ from vllm.v1.kv_cache_interface import (
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.request import Request
logger = init_logger(__name__)
class SingleTypeKVCacheManager(ABC):
"""
@@ -1210,6 +1214,10 @@ class CircularBufferManager(FullAttentionManager):
return 0
class KpoolTailManager(CircularBufferManager):
"""One-block circular scratch manager for ``KpoolTailSpec``."""
class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec, **kwargs) -> None:
super().__init__(kv_cache_spec, **kwargs)
@@ -2085,6 +2093,11 @@ def register_all_kvcache_specs(vllm_config):
SlidingWindowManager,
uniform_type_base_spec=SlidingWindowMLASpec,
)
KVCacheSpecRegistry.register(
KpoolTailSpec,
KpoolTailManager,
uniform_type_base_spec=KpoolTailSpec,
)
KVCacheSpecRegistry.register(
MambaSpec, MambaManager, uniform_type_base_spec=MambaSpec