[KV Connector] Support MooncakeStore with hybrid DCP prefix caching (#53324)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Signed-off-by: wzhao18 <weizha@nvidia.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Wei Zhao
2026-08-29 02:10:23 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 43196f2458
commit 7fd9cc036e
14 changed files with 560 additions and 85 deletions
@@ -109,11 +109,8 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
f"{cache_block_size} (mamba_cache_mode != 'align')"
)
pcp = vllm_config.parallel_config.prefill_context_parallel_size
dcp = vllm_config.parallel_config.decode_context_parallel_size
if len(kv_cache_config.transfer_groups) > 1 and pcp * dcp > 1:
unsupported.append(
f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention"
)
if len(kv_cache_config.transfer_groups) > 1 and pcp > 1:
unsupported.append(f"PCP > 1 (pcp={pcp}) with hybrid attention")
if unsupported:
raise ValueError(
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
@@ -55,6 +55,12 @@ class ExternalCachedBlockPool:
return [self._present_block] * len(group_ids)
return None
def contains(self, group_id: int, block_hash: BlockHash) -> bool:
"""Return whether a group has a loadable key for this hash."""
return (
self._exists is not None and (group_id, bytes(block_hash)) in self._exists
)
class MooncakeStoreCoordinator:
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
@@ -67,6 +73,7 @@ class MooncakeStoreCoordinator:
hash_block_size: int,
use_eagle: bool = False,
retention_interval: int | None = None,
dcp_world_size: int = 1,
) -> None:
assert all(
g.kv_cache_spec.block_size % hash_block_size == 0 for g in kv_cache_groups
@@ -83,7 +90,7 @@ class MooncakeStoreCoordinator:
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
self.enable_partial_hash_hits = partial_hash_hits_enabled(
kv_cache_groups, hash_block_size
kv_cache_groups, hash_block_size, dcp_world_size
)
self.use_eagle = use_eagle
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
@@ -376,10 +383,11 @@ class MooncakeStoreCoordinator:
# Truncate full-attention hit_blocks to final converged length;
# other specs already trim themselves inside their hit logic. cdiv keeps
# the partial tail block when hit_length is not block-aligned.
first_group = self.attention_groups[0]
if isinstance(first_group.spec, FullAttentionSpec):
num_blocks = cdiv(hit_length, first_group.spec.block_size)
for group_id in first_group.group_ids:
for group in self.attention_groups:
if not isinstance(group.spec, FullAttentionSpec):
continue
num_blocks = cdiv(hit_length, group.spec.block_size)
for group_id in group.group_ids:
full_blks = hit_blocks_by_group[group_id]
assert full_blks is not None
del full_blks[num_blocks:]
@@ -398,15 +406,17 @@ def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
def partial_hash_hits_enabled(
kv_cache_groups: list[KVCacheGroupSpec], hash_block_size: int
kv_cache_groups: list[KVCacheGroupSpec],
hash_block_size: int,
dcp_world_size: int = 1,
) -> bool:
"""Mirror of core's ``HybridKVCacheCoordinator.enable_partial_hash_hits``
(its dcp == 1 clause holds: the connector rejects hybrid + DCP/PCP > 1).
Single copy on purpose — scheduler and coordinator must not disagree.
"""
"""Match core's DCP-aware Mamba partial-hit condition."""
return any(
isinstance(spec := _unwrap_spec(g.kv_cache_spec), MambaSpec)
and spec.mamba_cache_mode == "align"
and spec.block_size > hash_block_size
and (
(dcp_world_size == 1 and spec.block_size > hash_block_size)
or (dcp_world_size > 1 and spec.block_size >= hash_block_size)
)
for g in kv_cache_groups
)
@@ -300,6 +300,39 @@ class ChunkedTokenDatabase:
yield start_idx, end_idx, h
@dataclass(frozen=True)
class TailKeyBoundary:
"""Hash boundary used to key a group's tail block in the store.
Attributes:
group_id: KV-cache group containing the tail block.
num_tokens: Token boundary whose prefix hash identifies the matched
stored block. The loader uses
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
hash implied by ``MooncakeLookupResult.hit_length``. This changes
only the load key, not the reusable prefix.
"""
group_id: int
num_tokens: int
@dataclass
class MooncakeLookupResult:
"""Lookup result used to build the subsequent load request.
Attributes:
hit_length: Longest prefix that every KV-cache group can reuse after
their individual cache hits converge.
tail_key_boundaries: Hash boundary used to store each cache group's
tail block when ``hit_length`` does not identify its store key.
There is one entry per group for every nonzero hit.
"""
hit_length: int
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
@dataclass
class LoadSpec:
"""Specification for loading KV cache from external store."""
@@ -308,6 +341,7 @@ class LoadSpec:
kvpool_cached_tokens: int
can_load: bool
token_len: int = 0
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
@dataclass
@@ -16,7 +16,9 @@ Wire format (REQ/REP over IPC):
fixed-size block hash (0 when there are no hashes)
frame 3: raw block hashes concatenated back-to-back (each hash_len
bytes); the server splits on hash_len
Response: [hit_count: u32 big-endian, 4 bytes]
Response: hit_length (u32 big-endian, first 4 bytes), followed by zero
or more 8-byte tail-key boundaries. Each entry is group_id
(u32), then boundary_tokens (u32).
msg_type == RESET_MSG:
(no payload frames)
@@ -31,6 +33,11 @@ Mirrors the named-tag convention used by the NIXL connector (see
``vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py``).
"""
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
MooncakeLookupResult,
TailKeyBoundary,
)
# Request message-type tags. Frame 0 of every request.
LOOKUP_MSG: bytes = b"lookup"
RESET_MSG: bytes = b"reset"
@@ -38,3 +45,37 @@ RESET_MSG: bytes = b"reset"
# Single-byte response status codes for admin commands.
RESP_OK: bytes = b"\x01"
RESP_ERR: bytes = b"\x00"
# group_id (u32), boundary_tokens (u32).
TAIL_KEY_BOUNDARY_ENTRY_SIZE: int = 8
def encode_lookup_response(result: MooncakeLookupResult) -> bytes:
hit_length = result.hit_length.to_bytes(4, "big")
if not result.tail_key_boundaries:
return hit_length
payload = bytearray(hit_length)
for boundary in result.tail_key_boundaries:
payload.extend(boundary.group_id.to_bytes(4, "big"))
payload.extend(boundary.num_tokens.to_bytes(4, "big"))
return bytes(payload)
def decode_lookup_response(payload: bytes) -> MooncakeLookupResult:
if len(payload) < 4 or (len(payload) - 4) % TAIL_KEY_BOUNDARY_ENTRY_SIZE:
raise ValueError("Invalid Mooncake lookup response")
hit_length = int.from_bytes(payload[:4], "big")
if len(payload) == 4:
return MooncakeLookupResult(hit_length)
boundaries = []
for offset in range(4, len(payload), TAIL_KEY_BOUNDARY_ENTRY_SIZE):
boundaries.append(
TailKeyBoundary(
group_id=int.from_bytes(payload[offset : offset + 4], "big"),
num_tokens=int.from_bytes(payload[offset + 4 : offset + 8], "big"),
)
)
return MooncakeLookupResult(hit_length, tuple(boundaries))
@@ -76,7 +76,9 @@ class MooncakeStoreScheduler:
kv_cache_config, vllm_config
)
self.enable_partial_hash_hits = partial_hash_hits_enabled(
kv_cache_config.kv_cache_groups, self._hash_block_size
kv_cache_config.kv_cache_groups,
self._hash_block_size,
vllm_config.parallel_config.decode_context_parallel_size,
)
# Per-request state
@@ -115,15 +117,16 @@ class MooncakeStoreScheduler:
if request.num_tokens < align:
return 0, False
num_external_hit_tokens = self.client.lookup(
lookup_result = self.client.lookup(
request.request_id,
request.num_tokens,
request.block_hashes,
non_block=self.lookup_async,
)
if num_external_hit_tokens is None:
if lookup_result is None:
# Lookup not ready yet; scheduler will retry on a later step.
return None, False
num_external_hit_tokens = lookup_result.hit_length
if num_external_hit_tokens < num_computed_tokens:
need_to_allocate = 0
@@ -145,6 +148,7 @@ class MooncakeStoreScheduler:
vllm_cached_tokens=num_computed_tokens,
kvpool_cached_tokens=num_external_hit_tokens,
can_load=False,
tail_key_boundaries=lookup_result.tail_key_boundaries,
)
return need_to_allocate, self.load_async
@@ -44,16 +44,20 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
BlobBlockHashes,
ChunkedTokenDatabase,
KeyMetadata,
MooncakeLookupResult,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
PoolKey,
ReqMeta,
TailKeyBoundary,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import ( # noqa: E501
LOOKUP_MSG,
RESET_MSG,
RESP_ERR,
RESP_OK,
decode_lookup_response,
encode_lookup_response,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
@@ -63,11 +67,11 @@ from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.kv_cache_utils import (
BlockHash,
maybe_convert_block_hash,
resolve_dcp_kv_cache_spec,
resolve_kv_cache_block_sizes,
)
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
@@ -1179,6 +1183,12 @@ class KVCacheStoreRecvingThread(KVTransferThread):
# Skip chunks the consumer's per-group spec wouldn't populate
# locally (e.g. SWA pre-window) even if the producer stored them.
load_mask_per_group = self.coord.load_mask(req_meta.block_hashes, token_len)
tail_key_boundaries = {
boundary.group_id: boundary.num_tokens
for boundary in (
req_meta.load_spec.tail_key_boundaries # type: ignore[union-attr]
)
}
addr_list: list[list[int]] = []
size_list: list[list[int]] = []
@@ -1193,6 +1203,13 @@ class KVCacheStoreRecvingThread(KVTransferThread):
chunk_idx = start // db.block_size
if chunk_idx >= len(mask) or not mask[chunk_idx]:
continue
boundary_tokens = (
tail_key_boundaries.get(g_idx) if end == token_len else None
)
if boundary_tokens is not None:
block_hash = req_meta.block_hashes[
boundary_tokens // db.hash_block_size - 1
]
key_list.append(db.key_for(block_hash))
chunks.append((start, end))
g_addrs, g_sizes, g_block_ids = db.prepare_values(
@@ -1494,21 +1511,15 @@ class MooncakeStoreWorker:
)
return
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
# self.block_size (= scheduler_block_size) so the coordinator's
# ``block_size % hash_block_size == 0`` invariant holds.
groups = list(kv_cache_config.transfer_groups)
if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size:
g = groups[0]
groups = [
dataclasses.replace(
g,
kv_cache_spec=dataclasses.replace(
g.kv_cache_spec, block_size=self.block_size
),
)
]
self._kv_cache_groups: list[KVCacheGroupSpec] = groups
self._kv_cache_groups = [
dataclasses.replace(
group,
kv_cache_spec=resolve_dcp_kv_cache_spec(
group.kv_cache_spec, self.dcp_size
),
)
for group in kv_cache_config.transfer_groups
]
spec_cfg = getattr(vllm_config, "speculative_config", None)
use_eagle = bool(
spec_cfg.use_eagle()
@@ -1521,6 +1532,7 @@ class MooncakeStoreWorker:
hash_block_size=self.hash_block_size,
use_eagle=use_eagle,
retention_interval=kv_cache_config.prefix_cache_retention_interval,
dcp_world_size=self.dcp_size,
)
# One ChunkedTokenDatabase per group; addresses populated in
# register_kv_caches once the kv-cache layout is known. Each group's
@@ -1881,7 +1893,9 @@ class MooncakeStoreWorker:
return None
return MooncakeStoreWorkerMetadata(completed_saves=completed_saves)
def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int:
def lookup(
self, num_tokens: int, block_hashes: Sequence[BlockHash]
) -> MooncakeLookupResult:
"""Check how many prefix tokens exist in the store.
Checks across all rank-specific key namespaces that may be loaded. A
@@ -1889,11 +1903,11 @@ class MooncakeStoreWorker:
the last token is recomputed for sampling.
"""
if self._capacity_only:
return 0
return MooncakeLookupResult(0)
token_len = self.coord.align_lookup_length(num_tokens)
if not block_hashes or token_len <= 0:
return 0
return MooncakeLookupResult(0)
# Build per-(group, hash) candidate keys expanded across rank namespaces.
# candidate_meta stores the (group, hash_bytes) for key slice.
@@ -1934,7 +1948,7 @@ class MooncakeStoreWorker:
candidate_meta.append((g_idx, bytes(h)))
if not candidate_keys:
return 0
return MooncakeLookupResult(0)
lookup_start = time.perf_counter()
try:
@@ -1953,7 +1967,7 @@ class MooncakeStoreWorker:
num_failed_keys=len(candidate_keys),
)
logger.error("Remote connection failed in lookup: %s", e)
return 0
return MooncakeLookupResult(0)
# A (group, hash) is "present" only when every namespace that will be
# loaded has it (per-group count: sharded groups need every rank's
@@ -1970,7 +1984,7 @@ class MooncakeStoreWorker:
self.hash_block_size,
exists_set,
)
_masks, hit_length = self.coord.find_longest_cache_hit(
_, hit_length = self.coord.find_longest_cache_hit(
block_hashes,
token_len,
cached_block_pool,
@@ -1978,13 +1992,63 @@ class MooncakeStoreWorker:
if hit_length >= num_tokens:
usable_length = self.coord.align_lookup_length(num_tokens - 1)
if usable_length <= 0:
return 0
_masks, hit_length = self.coord.find_longest_cache_hit(
return MooncakeLookupResult(0)
_, hit_length = self.coord.find_longest_cache_hit(
block_hashes,
usable_length,
cached_block_pool,
)
return hit_length
return MooncakeLookupResult(
hit_length,
self._tail_key_boundaries(
block_hashes,
hit_length,
cached_block_pool,
),
)
def _tail_key_boundaries(
self,
block_hashes: Sequence[BlockHash],
hit_length: int,
cached_block_pool: ExternalCachedBlockPool,
) -> tuple[TailKeyBoundary, ...]:
"""Return the hash boundary used to store each group's tail block.
With fine-grained prefix matching, ``hit_length`` may fall within a
physical cache block and may not align with the hash boundary used to
store that block. For each KV-cache group, return the token boundary
whose hash was used as the store key.
"""
if hit_length <= 0:
return ()
boundaries = []
hit_boundary_hash_idx = hit_length // self.hash_block_size - 1
for group_id, db in enumerate(self.token_dbs):
chunk_id = cdiv(hit_length, db.block_size) - 1
boundary_tokens = hit_length
contains_hit_boundary = cached_block_pool.contains(
group_id, block_hashes[hit_boundary_hash_idx]
)
if not self.coord.enable_partial_hash_hits:
assert contains_hit_boundary
if not contains_hit_boundary:
next_chunk_hash_idx = min(
(chunk_id + 1) * db.block_size // self.hash_block_size,
len(block_hashes),
)
for hash_idx in range(hit_boundary_hash_idx + 1, next_chunk_hash_idx):
if cached_block_pool.contains(group_id, block_hashes[hash_idx]):
boundary_tokens = (hash_idx + 1) * self.hash_block_size
break
else:
raise AssertionError(
f"No tail key found for cache group {group_id} at "
f"hit length {hit_length}"
)
boundaries.append(TailKeyBoundary(group_id, boundary_tokens))
return tuple(boundaries)
def get_kv_events(self) -> list[BlockStored]:
if self.enable_kv_events and self.kv_send_thread is not None:
@@ -2017,7 +2081,7 @@ class LookupKeyServer:
"""ZMQ server on worker rank 0 for the LookupKey admin channel.
Handles two request types, tagged at frame 0:
- ``LOOKUP_MSG``: prefix-cache hit query, returns hit count.
- ``LOOKUP_MSG``: prefix-cache hit query, returns its load plan.
- ``RESET_MSG``: drains the send thread queue, then runs
``store.remove_all(force=True)``. Caller must have paused the
scheduler first.
@@ -2054,7 +2118,7 @@ class LookupKeyServer:
blob = all_frames[3].buffer
block_hashes = BlobBlockHashes(blob, hash_len)
result = self.store_worker.lookup(num_tokens, block_hashes)
self.socket.send(result.to_bytes(4, "big"))
self.socket.send(encode_lookup_response(result))
elif msg_type == RESET_MSG:
try:
@@ -2114,9 +2178,11 @@ class LookupKeyClient:
self.executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="MooncakeLookupClient"
)
self.futures: dict[str, Future[int]] = {}
self.futures: dict[str, Future[MooncakeLookupResult]] = {}
def _lookup(self, num_tokens: int, block_hashes: list[BlockHash]) -> int:
def _lookup(
self, num_tokens: int, block_hashes: list[BlockHash]
) -> MooncakeLookupResult:
hash_len = len(block_hashes[0]) if block_hashes else 0
all_frames = (
LOOKUP_MSG,
@@ -2126,7 +2192,7 @@ class LookupKeyClient:
)
self.socket.send_multipart(all_frames, copy=False)
resp = self.socket.recv()
return int.from_bytes(resp, "big")
return decode_lookup_response(resp)
def lookup(
self,
@@ -2134,7 +2200,7 @@ class LookupKeyClient:
num_tokens: int,
block_hashes: list[BlockHash],
non_block: bool = False,
) -> int | None:
) -> MooncakeLookupResult | None:
"""If non_block is True, will return None until the result is ready,
so the caller retries on a later step."""
future = self.futures.get(req_id)
@@ -2147,7 +2213,7 @@ class LookupKeyClient:
return future.result()
except Exception as e:
logger.error("Async Mooncake lookup failed for %s: %s", req_id, e)
return 0
return MooncakeLookupResult(0)
finally:
del self.futures[req_id]