[Model] add GLM-5.3-Flash support (#53906)

This commit is contained in:
Jiangyun Zhu
2026-09-03 09:40:35 -07:00
committed by GitHub
parent da8ec28268
commit 98ed0856f3
94 changed files with 17927 additions and 266 deletions
@@ -56,6 +56,7 @@ from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
AttentionSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheSpec,
MambaSpec,
SlidingWindowSpec,
@@ -1675,7 +1676,9 @@ class MooncakeConnectorWorker:
block_len = region_cache.stride(0) * region_cache.element_size()
region_base_addresses.append(base_addr)
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
if isinstance(layer_spec, KpoolTailSpec):
kv_block_len = layer_spec.unpadded_page_size_bytes // 2
elif isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
assert (
layer_spec.page_size_bytes
% self._physical_blocks_per_logical_kv_block
@@ -75,8 +75,14 @@ class MooncakeStoreCoordinator:
retention_interval: int | None = None,
dcp_world_size: int = 1,
) -> None:
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
# to divide groups that participate in prefix caching. Non-shareable
# scratch groups (e.g. GLM-5.3-Flash's kpool tail) are skipped by
# _verify_and_split_kv_cache_groups and never probed for hits.
assert all(
g.kv_cache_spec.block_size % hash_block_size == 0 for g in kv_cache_groups
g.kv_cache_spec.block_size % hash_block_size == 0
for g in kv_cache_groups
if g.kv_cache_spec.prefix_cacheable
), "block_size must be divisible by hash_block_size"
assert scheduler_block_size % hash_block_size == 0, (
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
@@ -116,6 +122,12 @@ class MooncakeStoreCoordinator:
"""
attention_groups: list[SpecGroup] = []
for i, g in enumerate(self.kv_cache_groups):
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
# kpool tail): per-request scratch, never shareable, so they must
# not participate in hit lookup. Mirrors core's
# KVCacheCoordinator.verify_and_split_kv_cache_groups.
if not g.kv_cache_spec.prefix_cacheable:
continue
spec = _unwrap_spec(g.kv_cache_spec)
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None, (
@@ -523,6 +523,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
enable_group_semantics: bool = False,
supports_group_ids: bool = False,
record_operation: Callable[..., None] | None = None,
group_participates: Sequence[bool] | None = None,
):
super().__init__(
store,
@@ -537,6 +538,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
self.group_put_steps = group_put_steps
self.coord = coord
self.kv_role = kv_role
self.group_participates = (
list(group_participates)
if group_participates is not None
else [True] * len(token_databases)
)
# req_id -> ids of its store jobs that are still queued or running.
# Keying by store_job_id, which never repeats for the engine's lifetime,
# rather than counting jobs per request id makes the ledger immune to id
@@ -744,6 +750,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
saved = self._saved_offset.get(req_meta.req_id, 0)
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
for g_idx, db in enumerate(self.token_databases):
if not self.group_participates[g_idx]:
continue
group_blocks = req_meta.block_ids[g_idx]
# Distribute across ranks by the same rule as normal chunks.
put_step = self.group_put_steps[g_idx]
@@ -983,6 +991,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
group_indices: list[int] = []
store_shard_ids: list[StoreShardId] = []
for g_idx, db in enumerate(self.token_databases):
if not self.group_participates[g_idx]:
continue
# Rotate the stride phase per group to balance load across ranks.
put_step = self.group_put_steps[g_idx]
put_step_rank = (self.tp_rank + g_idx) % put_step
@@ -1253,6 +1263,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
disk_offload_buffer_budget_bytes: int | None = None,
record_operation: Callable[..., None] | None = None,
request_queue: queue.Queue[Any] | None = None,
group_participates: Sequence[bool] | None = None,
):
super().__init__(
store,
@@ -1264,6 +1275,11 @@ class KVCacheStoreRecvingThread(KVTransferThread):
record_operation=record_operation,
request_queue=request_queue,
)
self.group_participates = (
list(group_participates)
if group_participates is not None
else [True] * len(token_databases)
)
# _invalid_block_ids can be access by both the Worker and RecvingThread
self._invalid_block_ids_lock = threading.Lock()
self._invalid_block_ids: set[int] = set()
@@ -1311,6 +1327,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
key_list: list[str] = []
block_id_list: list[int] = []
for g_idx, db in enumerate(self.token_databases):
if not self.group_participates[g_idx]:
continue
mask = load_mask_per_group[g_idx]
chunks: list[tuple[int, int]] = []
store_shard_ids: list[StoreShardId] = []
@@ -1791,6 +1809,11 @@ class MooncakeStoreWorker:
"""Construct token databases and their Store layouts."""
token_dbs: list[ChunkedTokenDatabase] = []
for group_idx, group in enumerate(self._kv_cache_groups):
hash_block_size = (
self.hash_block_size
if group.kv_cache_spec.prefix_cacheable
else group.kv_cache_spec.block_size
)
group_tp_rank = self.tp_rank
if layout_cls is None:
group_tp_rank //= self._group_tp_replication_factors[group_idx]
@@ -1805,7 +1828,7 @@ class MooncakeStoreWorker:
store_layout = layout_cls(
group_metadata,
group.kv_cache_spec.block_size,
self.hash_block_size,
hash_block_size,
local_tp_size=self.tp_size,
store_tp_size=self.store_tp_size,
tp_rank=self.tp_rank,
@@ -1815,7 +1838,7 @@ class MooncakeStoreWorker:
ChunkedTokenDatabase(
group_metadata,
group.kv_cache_spec.block_size,
hash_block_size=self.hash_block_size,
hash_block_size=hash_block_size,
store_layout=store_layout,
)
)
@@ -1937,6 +1960,10 @@ class MooncakeStoreWorker:
enable_group_semantics=self.enable_group_semantics,
supports_group_ids=self._supports_group_ids,
record_operation=self._record_kv_connector_operation,
group_participates=[
group.kv_cache_spec.prefix_cacheable
for group in self._kv_cache_groups
],
)
self.kv_send_thread.start()
@@ -1954,6 +1981,10 @@ class MooncakeStoreWorker:
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
record_operation=self._record_kv_connector_operation,
request_queue=self.recv_request_queue,
group_participates=[
group.kv_cache_spec.prefix_cacheable
for group in self._kv_cache_groups
],
)
recv_thread.name = f"KVCacheStoreRecvingThread-{i}"
recv_thread.start()
@@ -2126,6 +2157,8 @@ class MooncakeStoreWorker:
fine_grained = self.coord.enable_partial_hash_hits
lookup_masks = None if fine_grained else self.coord.lookup_mask(token_len)
for g_idx, db in enumerate(self.token_dbs):
if not self._kv_cache_groups[g_idx].kv_cache_spec.prefix_cacheable:
continue
spec_block_size = db.block_size
key_prefixes = self._lookup_key_prefixes[g_idx]
if fine_grained:
@@ -2236,6 +2269,9 @@ class MooncakeStoreWorker:
boundaries = []
hit_boundary_hash_idx = hit_length // self.hash_block_size - 1
for group_id, db in enumerate(self.token_dbs):
if not self._kv_cache_groups[group_id].kv_cache_spec.prefix_cacheable:
# Scratch groups are never stored, so they have no tail key.
continue
chunk_id = cdiv(hit_length, db.block_size) - 1
boundary_tokens = hit_length
contains_hit_boundary = cached_block_pool.contains(
@@ -74,6 +74,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheLayout,
KVCacheSpec,
MambaSpec,
@@ -99,6 +100,46 @@ def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool:
return len(block_strides) == len(storage_ptrs) == 1
def _tensor_byte_span_end(cache: torch.Tensor) -> int:
"""Return the exclusive end address touched by a nonnegative-stride view."""
if cache.numel() == 0:
return cache.data_ptr()
if any(stride < 0 for stride in cache.stride()):
raise ValueError("NIXL cache views must have nonnegative strides")
max_element_offset = sum(
(size - 1) * stride for size, stride in zip(cache.shape, cache.stride())
)
return cache.data_ptr() + (max_element_offset + 1) * cache.element_size()
def _uses_dense_virtual_transfer_pages(
layer_spec: KVCacheSpec,
cache: torch.Tensor,
physical_page_size: int,
num_blocks: int,
) -> bool:
"""Return whether a compressed kernel view can be split into NIXL pages."""
if not (
isinstance(layer_spec, MLAAttentionSpec)
and layer_spec.tokens_per_state > 1
and cache.ndim == 4
and cache.shape[1] == 1
and cache.is_contiguous()
and physical_page_size > 0
and layer_spec.state_content_size_bytes > 0
):
return False
block_stride = cache.stride(0) * cache.element_size()
return (
block_stride > physical_page_size
and block_stride % physical_page_size == 0
and physical_page_size % layer_spec.state_content_size_bytes == 0
and cache.shape[0] * (block_stride // physical_page_size) == num_blocks
and cache.nbytes == num_blocks * physical_page_size
)
class NixlBaseConnectorWorker:
"""Base implementation of Worker side methods shared by pull and push."""
@@ -1150,6 +1191,31 @@ class NixlBaseConnectorWorker:
# aliases even though every view shares one block-major allocation.
packed_storage = packed_storage and not self._is_csa_linear
layer_specs: dict[str, KVCacheSpec] = {}
compressed_region_owners: dict[int, torch.Tensor] = {}
for layer_name, cache in xfer_buffers.items():
layer_spec = self._layer_specs.get(layer_name)
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
layer_spec = layer_spec.kv_cache_specs[layer_name]
if layer_spec is None:
continue
layer_specs[layer_name] = layer_spec
physical_page_size = (
layer_spec.page_size_bytes
if isinstance(layer_spec, MambaSpec)
else layer_spec.page_size_bytes
// self._physical_blocks_per_logical_kv_block
)
num_blocks = (
self._logical_num_blocks
if isinstance(layer_spec, MambaSpec)
else self.num_blocks
)
if _uses_dense_virtual_transfer_pages(
layer_spec, cache, physical_page_size, num_blocks
):
compressed_region_owners.setdefault(cache.data_ptr(), cache)
# K and V are packed into the content dim, so each attention layer is a
# single NIXL region whose block transfers as one unit. Mamba layers instead
# register separate conv/ssm sub-regions (see `_build_mamba_local`).
@@ -1158,7 +1224,7 @@ class NixlBaseConnectorWorker:
# kernel requires a specific block size. This leads to SSM and FA layers
# having different num_blocks.
# `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this.
layer_spec = self._layer_specs.get(layer_name)
layer_spec = layer_specs.get(layer_name)
if layer_spec is None:
logger.debug(
"Skipping layer %s as no KVCache spec is present. "
@@ -1166,9 +1232,6 @@ class NixlBaseConnectorWorker:
layer_name,
)
continue
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
# DSA Indexer case: UniformTypeKVCacheSpecs merges kv_cache_specs
layer_spec = layer_spec.kv_cache_specs[layer_name]
# `layer_spec.page_size_bytes` only accounts for logical page_size, that is
# the page_size assuming constant `self._logical_num_blocks`.
physical_page_size = (
@@ -1187,6 +1250,33 @@ class NixlBaseConnectorWorker:
)
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
if isinstance(layer_spec, KpoolTailSpec):
compressed_owner = compressed_region_owners.get(cache.data_ptr())
if compressed_owner is not None:
owner_storage = compressed_owner.untyped_storage()
owner_end = compressed_owner.data_ptr() + compressed_owner.nbytes
tail_is_covered = (
compressed_owner.is_contiguous()
and owner_storage.data_ptr() == storage_addr
and _tensor_byte_span_end(cache) <= owner_end
)
if not tail_is_covered:
raise AssertionError(
"Kpool tail cache is not fully covered by its compressed "
f"indexer region: layer={layer_name}, "
f"tail_shape={tuple(cache.shape)}, "
f"tail_stride={tuple(cache.stride())}, "
f"owner_shape={tuple(compressed_owner.shape)}, "
f"owner_stride={tuple(compressed_owner.stride())}"
)
logger.debug(
"Skipping layer %s because its compressed indexer region "
"covers the same storage",
layer_name,
)
continue
# Memory registration follows allocations, while transfer regions follow
# logical layers (or contiguous head segments). This keeps strided
# cross-layer views inside their registered allocation.
@@ -1228,7 +1318,15 @@ class NixlBaseConnectorWorker:
and cache.stride(2) == cache.shape[3]
and cache.stride(1) == cache.shape[2] * cache.shape[3]
)
if storage_is_block_major and (
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
layer_spec, cache, physical_page_size, num_blocks
)
if virtual_transfer_pages:
# A compressed kernel row can contain multiple NIXL transfer pages.
region_specs = [
(cache.data_ptr(), physical_page_size, physical_page_size)
]
elif storage_is_block_major and (
(packed_storage and is_mla_region)
or (not hnc_contiguous and not self._is_csa_linear)
):
@@ -1243,7 +1341,15 @@ class NixlBaseConnectorWorker:
]
else:
segment_bytes = num_blocks * block_stride
assert cache.nbytes % segment_bytes == 0
if cache.nbytes % segment_bytes != 0:
raise AssertionError(
"KV cache view cannot be partitioned into NIXL regions: "
f"layer={layer_name}, cache_nbytes={cache.nbytes}, "
f"num_blocks={num_blocks}, block_stride={block_stride}, "
f"physical_page_size={physical_page_size}, "
f"cache_shape={tuple(cache.shape)}, "
f"cache_stride={tuple(cache.stride())}"
)
num_segments = cache.nbytes // segment_bytes
region_block_len = (
block_stride if num_segments > 1 else physical_page_size
@@ -45,8 +45,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
# 9: Add block_strides
# 10: Add dense virtual transfer pages for compressed MLA caches
#
NIXL_CONNECTOR_VERSION: int = 9
NIXL_CONNECTOR_VERSION: int = 10
@dataclass