[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -56,6 +56,7 @@ from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
@@ -1675,7 +1676,9 @@ class MooncakeConnectorWorker:
|
||||
block_len = region_cache.stride(0) * region_cache.element_size()
|
||||
region_base_addresses.append(base_addr)
|
||||
|
||||
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
|
||||
if isinstance(layer_spec, KpoolTailSpec):
|
||||
kv_block_len = layer_spec.unpadded_page_size_bytes // 2
|
||||
elif isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
|
||||
assert (
|
||||
layer_spec.page_size_bytes
|
||||
% self._physical_blocks_per_logical_kv_block
|
||||
|
||||
@@ -75,8 +75,14 @@ class MooncakeStoreCoordinator:
|
||||
retention_interval: int | None = None,
|
||||
dcp_world_size: int = 1,
|
||||
) -> None:
|
||||
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
|
||||
# to divide groups that participate in prefix caching. Non-shareable
|
||||
# scratch groups (e.g. GLM-5.3-Flash's kpool tail) are skipped by
|
||||
# _verify_and_split_kv_cache_groups and never probed for hits.
|
||||
assert all(
|
||||
g.kv_cache_spec.block_size % hash_block_size == 0 for g in kv_cache_groups
|
||||
g.kv_cache_spec.block_size % hash_block_size == 0
|
||||
for g in kv_cache_groups
|
||||
if g.kv_cache_spec.prefix_cacheable
|
||||
), "block_size must be divisible by hash_block_size"
|
||||
assert scheduler_block_size % hash_block_size == 0, (
|
||||
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
|
||||
@@ -116,6 +122,12 @@ class MooncakeStoreCoordinator:
|
||||
"""
|
||||
attention_groups: list[SpecGroup] = []
|
||||
for i, g in enumerate(self.kv_cache_groups):
|
||||
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail): per-request scratch, never shareable, so they must
|
||||
# not participate in hit lookup. Mirrors core's
|
||||
# KVCacheCoordinator.verify_and_split_kv_cache_groups.
|
||||
if not g.kv_cache_spec.prefix_cacheable:
|
||||
continue
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None, (
|
||||
|
||||
@@ -523,6 +523,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
enable_group_semantics: bool = False,
|
||||
supports_group_ids: bool = False,
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
group_participates: Sequence[bool] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
@@ -537,6 +538,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self.group_put_steps = group_put_steps
|
||||
self.coord = coord
|
||||
self.kv_role = kv_role
|
||||
self.group_participates = (
|
||||
list(group_participates)
|
||||
if group_participates is not None
|
||||
else [True] * len(token_databases)
|
||||
)
|
||||
# req_id -> ids of its store jobs that are still queued or running.
|
||||
# Keying by store_job_id, which never repeats for the engine's lifetime,
|
||||
# rather than counting jobs per request id makes the ledger immune to id
|
||||
@@ -744,6 +750,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
saved = self._saved_offset.get(req_meta.req_id, 0)
|
||||
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
if not self.group_participates[g_idx]:
|
||||
continue
|
||||
group_blocks = req_meta.block_ids[g_idx]
|
||||
# Distribute across ranks by the same rule as normal chunks.
|
||||
put_step = self.group_put_steps[g_idx]
|
||||
@@ -983,6 +991,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
group_indices: list[int] = []
|
||||
store_shard_ids: list[StoreShardId] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
if not self.group_participates[g_idx]:
|
||||
continue
|
||||
# Rotate the stride phase per group to balance load across ranks.
|
||||
put_step = self.group_put_steps[g_idx]
|
||||
put_step_rank = (self.tp_rank + g_idx) % put_step
|
||||
@@ -1253,6 +1263,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
request_queue: queue.Queue[Any] | None = None,
|
||||
group_participates: Sequence[bool] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
@@ -1264,6 +1275,11 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
record_operation=record_operation,
|
||||
request_queue=request_queue,
|
||||
)
|
||||
self.group_participates = (
|
||||
list(group_participates)
|
||||
if group_participates is not None
|
||||
else [True] * len(token_databases)
|
||||
)
|
||||
# _invalid_block_ids can be access by both the Worker and RecvingThread
|
||||
self._invalid_block_ids_lock = threading.Lock()
|
||||
self._invalid_block_ids: set[int] = set()
|
||||
@@ -1311,6 +1327,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
key_list: list[str] = []
|
||||
block_id_list: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
if not self.group_participates[g_idx]:
|
||||
continue
|
||||
mask = load_mask_per_group[g_idx]
|
||||
chunks: list[tuple[int, int]] = []
|
||||
store_shard_ids: list[StoreShardId] = []
|
||||
@@ -1791,6 +1809,11 @@ class MooncakeStoreWorker:
|
||||
"""Construct token databases and their Store layouts."""
|
||||
token_dbs: list[ChunkedTokenDatabase] = []
|
||||
for group_idx, group in enumerate(self._kv_cache_groups):
|
||||
hash_block_size = (
|
||||
self.hash_block_size
|
||||
if group.kv_cache_spec.prefix_cacheable
|
||||
else group.kv_cache_spec.block_size
|
||||
)
|
||||
group_tp_rank = self.tp_rank
|
||||
if layout_cls is None:
|
||||
group_tp_rank //= self._group_tp_replication_factors[group_idx]
|
||||
@@ -1805,7 +1828,7 @@ class MooncakeStoreWorker:
|
||||
store_layout = layout_cls(
|
||||
group_metadata,
|
||||
group.kv_cache_spec.block_size,
|
||||
self.hash_block_size,
|
||||
hash_block_size,
|
||||
local_tp_size=self.tp_size,
|
||||
store_tp_size=self.store_tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
@@ -1815,7 +1838,7 @@ class MooncakeStoreWorker:
|
||||
ChunkedTokenDatabase(
|
||||
group_metadata,
|
||||
group.kv_cache_spec.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
store_layout=store_layout,
|
||||
)
|
||||
)
|
||||
@@ -1937,6 +1960,10 @@ class MooncakeStoreWorker:
|
||||
enable_group_semantics=self.enable_group_semantics,
|
||||
supports_group_ids=self._supports_group_ids,
|
||||
record_operation=self._record_kv_connector_operation,
|
||||
group_participates=[
|
||||
group.kv_cache_spec.prefix_cacheable
|
||||
for group in self._kv_cache_groups
|
||||
],
|
||||
)
|
||||
self.kv_send_thread.start()
|
||||
|
||||
@@ -1954,6 +1981,10 @@ class MooncakeStoreWorker:
|
||||
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
|
||||
record_operation=self._record_kv_connector_operation,
|
||||
request_queue=self.recv_request_queue,
|
||||
group_participates=[
|
||||
group.kv_cache_spec.prefix_cacheable
|
||||
for group in self._kv_cache_groups
|
||||
],
|
||||
)
|
||||
recv_thread.name = f"KVCacheStoreRecvingThread-{i}"
|
||||
recv_thread.start()
|
||||
@@ -2126,6 +2157,8 @@ class MooncakeStoreWorker:
|
||||
fine_grained = self.coord.enable_partial_hash_hits
|
||||
lookup_masks = None if fine_grained else self.coord.lookup_mask(token_len)
|
||||
for g_idx, db in enumerate(self.token_dbs):
|
||||
if not self._kv_cache_groups[g_idx].kv_cache_spec.prefix_cacheable:
|
||||
continue
|
||||
spec_block_size = db.block_size
|
||||
key_prefixes = self._lookup_key_prefixes[g_idx]
|
||||
if fine_grained:
|
||||
@@ -2236,6 +2269,9 @@ class MooncakeStoreWorker:
|
||||
boundaries = []
|
||||
hit_boundary_hash_idx = hit_length // self.hash_block_size - 1
|
||||
for group_id, db in enumerate(self.token_dbs):
|
||||
if not self._kv_cache_groups[group_id].kv_cache_spec.prefix_cacheable:
|
||||
# Scratch groups are never stored, so they have no tail key.
|
||||
continue
|
||||
chunk_id = cdiv(hit_length, db.block_size) - 1
|
||||
boundary_tokens = hit_length
|
||||
contains_hit_boundary = cached_block_pool.contains(
|
||||
|
||||
@@ -74,6 +74,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
@@ -99,6 +100,46 @@ def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool:
|
||||
return len(block_strides) == len(storage_ptrs) == 1
|
||||
|
||||
|
||||
def _tensor_byte_span_end(cache: torch.Tensor) -> int:
|
||||
"""Return the exclusive end address touched by a nonnegative-stride view."""
|
||||
if cache.numel() == 0:
|
||||
return cache.data_ptr()
|
||||
if any(stride < 0 for stride in cache.stride()):
|
||||
raise ValueError("NIXL cache views must have nonnegative strides")
|
||||
max_element_offset = sum(
|
||||
(size - 1) * stride for size, stride in zip(cache.shape, cache.stride())
|
||||
)
|
||||
return cache.data_ptr() + (max_element_offset + 1) * cache.element_size()
|
||||
|
||||
|
||||
def _uses_dense_virtual_transfer_pages(
|
||||
layer_spec: KVCacheSpec,
|
||||
cache: torch.Tensor,
|
||||
physical_page_size: int,
|
||||
num_blocks: int,
|
||||
) -> bool:
|
||||
"""Return whether a compressed kernel view can be split into NIXL pages."""
|
||||
if not (
|
||||
isinstance(layer_spec, MLAAttentionSpec)
|
||||
and layer_spec.tokens_per_state > 1
|
||||
and cache.ndim == 4
|
||||
and cache.shape[1] == 1
|
||||
and cache.is_contiguous()
|
||||
and physical_page_size > 0
|
||||
and layer_spec.state_content_size_bytes > 0
|
||||
):
|
||||
return False
|
||||
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
return (
|
||||
block_stride > physical_page_size
|
||||
and block_stride % physical_page_size == 0
|
||||
and physical_page_size % layer_spec.state_content_size_bytes == 0
|
||||
and cache.shape[0] * (block_stride // physical_page_size) == num_blocks
|
||||
and cache.nbytes == num_blocks * physical_page_size
|
||||
)
|
||||
|
||||
|
||||
class NixlBaseConnectorWorker:
|
||||
"""Base implementation of Worker side methods shared by pull and push."""
|
||||
|
||||
@@ -1150,6 +1191,31 @@ class NixlBaseConnectorWorker:
|
||||
# aliases even though every view shares one block-major allocation.
|
||||
packed_storage = packed_storage and not self._is_csa_linear
|
||||
|
||||
layer_specs: dict[str, KVCacheSpec] = {}
|
||||
compressed_region_owners: dict[int, torch.Tensor] = {}
|
||||
for layer_name, cache in xfer_buffers.items():
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
|
||||
layer_spec = layer_spec.kv_cache_specs[layer_name]
|
||||
if layer_spec is None:
|
||||
continue
|
||||
layer_specs[layer_name] = layer_spec
|
||||
physical_page_size = (
|
||||
layer_spec.page_size_bytes
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
num_blocks = (
|
||||
self._logical_num_blocks
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else self.num_blocks
|
||||
)
|
||||
if _uses_dense_virtual_transfer_pages(
|
||||
layer_spec, cache, physical_page_size, num_blocks
|
||||
):
|
||||
compressed_region_owners.setdefault(cache.data_ptr(), cache)
|
||||
|
||||
# K and V are packed into the content dim, so each attention layer is a
|
||||
# single NIXL region whose block transfers as one unit. Mamba layers instead
|
||||
# register separate conv/ssm sub-regions (see `_build_mamba_local`).
|
||||
@@ -1158,7 +1224,7 @@ class NixlBaseConnectorWorker:
|
||||
# kernel requires a specific block size. This leads to SSM and FA layers
|
||||
# having different num_blocks.
|
||||
# `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this.
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
layer_spec = layer_specs.get(layer_name)
|
||||
if layer_spec is None:
|
||||
logger.debug(
|
||||
"Skipping layer %s as no KVCache spec is present. "
|
||||
@@ -1166,9 +1232,6 @@ class NixlBaseConnectorWorker:
|
||||
layer_name,
|
||||
)
|
||||
continue
|
||||
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
|
||||
# DSA Indexer case: UniformTypeKVCacheSpecs merges kv_cache_specs
|
||||
layer_spec = layer_spec.kv_cache_specs[layer_name]
|
||||
# `layer_spec.page_size_bytes` only accounts for logical page_size, that is
|
||||
# the page_size assuming constant `self._logical_num_blocks`.
|
||||
physical_page_size = (
|
||||
@@ -1187,6 +1250,33 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
|
||||
if isinstance(layer_spec, KpoolTailSpec):
|
||||
compressed_owner = compressed_region_owners.get(cache.data_ptr())
|
||||
if compressed_owner is not None:
|
||||
owner_storage = compressed_owner.untyped_storage()
|
||||
owner_end = compressed_owner.data_ptr() + compressed_owner.nbytes
|
||||
tail_is_covered = (
|
||||
compressed_owner.is_contiguous()
|
||||
and owner_storage.data_ptr() == storage_addr
|
||||
and _tensor_byte_span_end(cache) <= owner_end
|
||||
)
|
||||
if not tail_is_covered:
|
||||
raise AssertionError(
|
||||
"Kpool tail cache is not fully covered by its compressed "
|
||||
f"indexer region: layer={layer_name}, "
|
||||
f"tail_shape={tuple(cache.shape)}, "
|
||||
f"tail_stride={tuple(cache.stride())}, "
|
||||
f"owner_shape={tuple(compressed_owner.shape)}, "
|
||||
f"owner_stride={tuple(compressed_owner.stride())}"
|
||||
)
|
||||
logger.debug(
|
||||
"Skipping layer %s because its compressed indexer region "
|
||||
"covers the same storage",
|
||||
layer_name,
|
||||
)
|
||||
continue
|
||||
|
||||
# Memory registration follows allocations, while transfer regions follow
|
||||
# logical layers (or contiguous head segments). This keeps strided
|
||||
# cross-layer views inside their registered allocation.
|
||||
@@ -1228,7 +1318,15 @@ class NixlBaseConnectorWorker:
|
||||
and cache.stride(2) == cache.shape[3]
|
||||
and cache.stride(1) == cache.shape[2] * cache.shape[3]
|
||||
)
|
||||
if storage_is_block_major and (
|
||||
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
|
||||
layer_spec, cache, physical_page_size, num_blocks
|
||||
)
|
||||
if virtual_transfer_pages:
|
||||
# A compressed kernel row can contain multiple NIXL transfer pages.
|
||||
region_specs = [
|
||||
(cache.data_ptr(), physical_page_size, physical_page_size)
|
||||
]
|
||||
elif storage_is_block_major and (
|
||||
(packed_storage and is_mla_region)
|
||||
or (not hnc_contiguous and not self._is_csa_linear)
|
||||
):
|
||||
@@ -1243,7 +1341,15 @@ class NixlBaseConnectorWorker:
|
||||
]
|
||||
else:
|
||||
segment_bytes = num_blocks * block_stride
|
||||
assert cache.nbytes % segment_bytes == 0
|
||||
if cache.nbytes % segment_bytes != 0:
|
||||
raise AssertionError(
|
||||
"KV cache view cannot be partitioned into NIXL regions: "
|
||||
f"layer={layer_name}, cache_nbytes={cache.nbytes}, "
|
||||
f"num_blocks={num_blocks}, block_stride={block_stride}, "
|
||||
f"physical_page_size={physical_page_size}, "
|
||||
f"cache_shape={tuple(cache.shape)}, "
|
||||
f"cache_stride={tuple(cache.stride())}"
|
||||
)
|
||||
num_segments = cache.nbytes // segment_bytes
|
||||
region_block_len = (
|
||||
block_stride if num_segments > 1 else physical_page_size
|
||||
|
||||
@@ -45,8 +45,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
|
||||
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
|
||||
# 9: Add block_strides
|
||||
# 10: Add dense virtual transfer pages for compressed MLA caches
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 9
|
||||
NIXL_CONNECTOR_VERSION: int = 10
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user