[KVConnector][DSV4] HMA support for Mooncake store connector (#42828)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-05-19 01:07:46 -07:00
committed by GitHub
parent f34623bf3c
commit 056bc2e166
10 changed files with 1834 additions and 445 deletions
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
SupportsHMA,
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
@@ -76,7 +77,7 @@ class MooncakeStoreKVEvents(KVConnectorKVEvents):
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
class MooncakeStoreConnector(KVConnectorBase_V1):
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
"""KV connector using MooncakeDistributedStore as shared KV pool."""
@property
@@ -87,6 +88,36 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
== "true"
)
@staticmethod
def _validate_kv_cache_config(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> None:
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
unsupported: list[str] = []
cache_block_size = vllm_config.cache_config.block_size
for g_idx, g in enumerate(kv_cache_config.kv_cache_groups):
spec = g.kv_cache_spec
if isinstance(spec, CrossAttentionSpec):
unsupported.append(f"group {g_idx}: CrossAttentionSpec")
# Enforce Mamba align mode
if isinstance(spec, MambaSpec) and spec.block_size != cache_block_size:
unsupported.append(
f"group {g_idx}: MambaSpec with block_size="
f"{spec.block_size} != cache_config.block_size="
f"{cache_block_size} (mamba_cache_mode != 'align')"
)
pcp = vllm_config.parallel_config.prefill_context_parallel_size
dcp = vllm_config.parallel_config.decode_context_parallel_size
if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1:
unsupported.append(
f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention"
)
if unsupported:
raise ValueError(
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
)
def __init__(
self,
vllm_config: VllmConfig,
@@ -99,6 +130,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
kv_cache_config=kv_cache_config, # type: ignore[arg-type]
)
assert vllm_config.kv_transfer_config is not None
assert kv_cache_config is not None, "kv_cache_config is required"
self._validate_kv_cache_config(vllm_config, kv_cache_config)
self._kv_cache_config = kv_cache_config
self.kv_role = vllm_config.kv_transfer_config.kv_role
self._kv_cache_events: MooncakeStoreKVEvents | None = None
@@ -106,9 +140,11 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
self.connector_worker: MooncakeStoreWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = MooncakeStoreScheduler(vllm_config)
self.connector_scheduler = MooncakeStoreScheduler(
vllm_config, kv_cache_config
)
else:
self.connector_worker = MooncakeStoreWorker(vllm_config)
self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config)
# ============================================================
# Scheduler-side methods
@@ -146,6 +182,13 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
self,
request: Request,
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
return self.request_finished_all_groups(request, (block_ids,))
def request_finished_all_groups(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
@@ -184,6 +227,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
self, kv_cache: torch.Tensor, attn_backend: type
):
assert self.connector_worker is not None
assert (
self._kv_cache_config is not None
and len(self._kv_cache_config.kv_cache_groups) == 1
), "Cross-layer KV cache does not supported with hybrid models"
self.connector_worker.register_cross_layers_kv_caches(kv_cache)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
@@ -0,0 +1,290 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
from typing import cast
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashList,
BlockHashListWithBlockSize,
KVCacheBlock,
)
from vllm.v1.core.single_type_kv_cache_manager import (
SingleTypeKVCacheManager,
spec_manager_map,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
KVCacheSpec,
UniformTypeKVCacheSpecs,
)
# Dummy placeholder hash for store_mask's template computation.
_DUMMY_BLOCK_HASH = BlockHash(b"\x00" * 32)
class ExternalCachedBlockPool:
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
def __init__(self, exists: set[tuple[int, bytes]] | None = None) -> None:
# ``exists=None`` is used on the recv side where hit_length is already
# determined and we just want each spec's manager to apply its own mask.
self._exists = exists
self.null_block = KVCacheBlock(block_id=0)
# Dummy ID 1 for present block for duck-typing.
self._present_block = KVCacheBlock(block_id=1)
def get_cached_block(
self,
block_hash: BlockHash,
group_ids: list[int],
) -> list[KVCacheBlock] | None:
# Mirrors BlockPool.get_cached_block: hit only when every group_id
# (groups sharing a spec) has the hash cached.
if self._exists is None:
return [self._present_block] * len(group_ids)
h = bytes(block_hash)
if all((g, h) in self._exists for g in group_ids):
return [self._present_block] * len(group_ids)
return None
class MooncakeStoreCoordinator:
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
``ExternalCachedBlockPool``."""
def __init__(
self,
kv_cache_groups: list[KVCacheGroupSpec],
scheduler_block_size: int,
hash_block_size: int,
use_eagle: bool = False,
) -> None:
assert all(
g.kv_cache_spec.block_size % hash_block_size == 0 for g in kv_cache_groups
), "block_size must be divisible by hash_block_size"
assert scheduler_block_size % hash_block_size == 0, (
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
f"hash_block_size ({hash_block_size})"
)
assert all(
scheduler_block_size % g.kv_cache_spec.block_size == 0
for g in kv_cache_groups
), "scheduler_block_size must be a multiple of each group's block_size"
self.kv_cache_groups = kv_cache_groups
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
self.use_eagle = use_eagle
self._verify_and_split_kv_cache_groups()
def _verify_and_split_kv_cache_groups(self) -> None:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
dispatches via spec_manager_map (we don't allocate managers).
"""
attention_groups: list[
tuple[KVCacheSpec, list[int], type[SingleTypeKVCacheManager]]
] = []
for i, g in enumerate(self.kv_cache_groups):
spec = _unwrap_spec(g.kv_cache_spec)
manager_cls = spec_manager_map[type(spec)]
for existing_spec, group_ids, existing_cls in attention_groups:
if existing_spec == spec:
assert manager_cls is existing_cls
group_ids.append(i)
break
else:
attention_groups.append((spec, [i], manager_cls))
# Full attention first (matches upstream convergence ordering).
self.attention_groups = sorted(
attention_groups,
key=lambda x: not isinstance(x[0], FullAttentionSpec),
)
self.eagle_attn_group_indices: set[int] = {
i
for i, (_, group_ids, _) in enumerate(self.attention_groups)
if any(self.kv_cache_groups[gid].is_eagle_group for gid in group_ids)
}
if self.use_eagle and not self.eagle_attn_group_indices:
self.eagle_attn_group_indices = set(range(len(self.attention_groups)))
def find_longest_cache_hit(
self,
block_hashes: list[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
) -> tuple[tuple[list[bool], ...], int]:
"""Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff
group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only);
recv-side callers skip False slots."""
blocks_per_group, hit_length = self._find_hit_blocks(
block_hashes, max_length, cached_block_pool
)
masks = tuple(
[blk is not cached_block_pool.null_block for blk in blocks]
for blocks in blocks_per_group
)
return masks, hit_length
def load_mask(
self,
block_hashes: list[BlockHash],
token_len: int,
) -> tuple[list[bool], ...]:
"""Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s
spec would populate chunk ``i`` locally at length ``token_len``
(e.g. SWA / Mamba tail-only).
"""
masks, _ = self.find_longest_cache_hit(
block_hashes, token_len, ExternalCachedBlockPool()
)
return masks
def store_mask(self, aligned_token_len: int) -> tuple[list[bool], ...]:
"""Per-group store masks: ``mask[g][i]`` is True iff chunk ``i`` of
group ``g`` would be populated by some future cache hit at length
``L = N * lcm_block_size <= aligned_token_len``.
"""
assert aligned_token_len % self.lcm_block_size == 0, (
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
f"lcm_block_size ({self.lcm_block_size})"
)
if aligned_token_len == 0:
return tuple([] for _ in self.kv_cache_groups)
num_chunks_per_group = [
aligned_token_len // g.kv_cache_spec.block_size
for g in self.kv_cache_groups
]
# Fast path: single group or full attn groups or uniform block_sizes
if all(
isinstance(spec, FullAttentionSpec)
or spec.block_size == self.lcm_block_size
for spec, _, _ in self.attention_groups
):
return tuple([True] * n for n in num_chunks_per_group)
n_segments = aligned_token_len // self.lcm_block_size
dummy_hashes: list[BlockHash] = [_DUMMY_BLOCK_HASH] * (
self.lcm_block_size // self.hash_block_size
)
template_masks, _ = self.find_longest_cache_hit(
dummy_hashes,
max_length=self.lcm_block_size,
cached_block_pool=ExternalCachedBlockPool(),
)
return tuple(
list(template_masks[g]) * n_segments
for g in range(len(self.kv_cache_groups))
)
def block_hashes_for_spec(
self, block_hashes: list[BlockHash], spec: KVCacheSpec
) -> BlockHashList:
if spec.block_size == self.hash_block_size:
return block_hashes
return BlockHashListWithBlockSize(
block_hashes, self.hash_block_size, spec.block_size
)
def _find_hit_blocks(
self,
block_hashes: list[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
"""Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but
dispatches via spec_manager_map (we don't allocate managers).
"""
if len(self.attention_groups) == 1:
spec, group_ids, manager_cls = self.attention_groups[0]
hashes = self.block_hashes_for_spec(block_hashes, spec)
hit_blocks = manager_cls.find_longest_cache_hit(
block_hashes=hashes,
max_length=max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
use_eagle=(0 in self.eagle_attn_group_indices),
alignment_tokens=spec.block_size,
)
num_groups = len(self.kv_cache_groups)
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
for gid, blks in zip(group_ids, hit_blocks, strict=True):
blocks_by_group[gid] = blks
return tuple(blocks_by_group), len(hit_blocks[0]) * spec.block_size
num_groups = len(self.kv_cache_groups)
hit_length = max_length
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
self.attention_groups[0][0], FullAttentionSpec
)
eagle_verified: set[int] = set()
while True:
curr_hit_length = hit_length
for idx, (spec, group_ids, manager_cls) in enumerate(self.attention_groups):
cached = hit_blocks_by_group[group_ids[0]]
if isinstance(spec, FullAttentionSpec) and cached is not None:
curr_hit_length = (
curr_hit_length // spec.block_size * spec.block_size
)
continue
use_eagle = (
idx in self.eagle_attn_group_indices and idx not in eagle_verified
)
_max_length = curr_hit_length
if use_eagle:
_max_length = min(curr_hit_length + spec.block_size, max_length)
hashes = self.block_hashes_for_spec(block_hashes, spec)
hit_blocks = manager_cls.find_longest_cache_hit(
block_hashes=hashes,
max_length=_max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
use_eagle=use_eagle,
alignment_tokens=self.lcm_block_size,
)
_new_hit_length = len(hit_blocks[0]) * spec.block_size
if use_eagle:
eagle_verified.add(idx)
elif _new_hit_length < curr_hit_length:
eagle_verified.clear()
curr_hit_length = _new_hit_length
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
hit_blocks_by_group[gid] = blocks
if curr_hit_length >= hit_length:
break
hit_length = curr_hit_length
if is_simple_hybrid:
break
# Truncate full-attention hit_blocks to final converged length;
# other specs already trim themselves inside their hit logic.
spec0, group_ids0, _ = self.attention_groups[0]
if isinstance(spec0, FullAttentionSpec):
num_blocks = hit_length // spec0.block_size
for gid in group_ids0:
full_blks = hit_blocks_by_group[gid]
assert full_blks is not None
del full_blks[num_blocks:]
return (
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
hit_length,
)
def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
if isinstance(spec, UniformTypeKVCacheSpecs):
return next(iter(spec.kv_cache_specs.values()))
return spec
@@ -15,7 +15,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashListWithBlockSize,
)
logger = init_logger(__name__)
@@ -29,6 +32,7 @@ class KeyMetadata:
pcp_rank: int
dcp_rank: int
pp_rank: int
group_id: int = 0
@dataclass(order=True)
@@ -46,6 +50,7 @@ class PoolKey:
self.key_metadata.pcp_rank,
self.key_metadata.dcp_rank,
self.key_metadata.pp_rank,
self.key_metadata.group_id,
self.chunk_hash,
)
)
@@ -57,6 +62,7 @@ class PoolKey:
f"@pcp{self.key_metadata.pcp_rank}"
f"@dcp{self.key_metadata.dcp_rank}"
f"@pp_rank:{self.key_metadata.pp_rank}"
f"@group:{self.key_metadata.group_id}"
f"@{self.chunk_hash}"
)
@@ -64,9 +70,20 @@ class PoolKey:
class ChunkedTokenDatabase:
"""Maps token positions to store keys and GPU memory addresses."""
def __init__(self, metadata: KeyMetadata, block_size: int):
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int | None = None,
):
self.metadata = metadata
self.block_size = block_size
self.hash_block_size = hash_block_size or block_size
if self.block_size % self.hash_block_size != 0:
raise ValueError(
f"block_size ({self.block_size}) must be a multiple of "
f"hash_block_size ({self.hash_block_size})"
)
self.kv_caches_base_addr: list[int] = []
self.block_len: list[int] = []
@@ -77,9 +94,6 @@ class ChunkedTokenDatabase:
self.kv_caches_base_addr = kv_caches_base_addr
def set_block_len(self, block_len: list[int]):
for length in block_len:
if length % self.block_size != 0:
raise ValueError(f"block_len {length} % {self.block_size} != 0")
self.block_len = block_len
def prepare_value(
@@ -96,7 +110,8 @@ class ChunkedTokenDatabase:
length = len(self.block_len)
for index, base_addr in enumerate(self.kv_caches_base_addr):
addr = base_addr + block_id * self.block_len[index % length]
size = self.block_len[index % length] // self.block_size * (end - start)
assert (end - start) % self.block_size == 0
size = self.block_len[index % length] * cdiv(end - start, self.block_size)
addr_list.append(addr)
size_list.append(size)
return addr_list, size_list, block_id
@@ -104,38 +119,34 @@ class ChunkedTokenDatabase:
def process_tokens(
self,
token_len: int,
block_hashes: list[BlockHash] | list[str],
block_hashes: list[BlockHash],
mask_num: int = 0,
) -> Iterable[tuple[int, int, PoolKey]]:
"""Process tokens and yield (start_idx, end_idx, pool_key) tuples.
Args:
token_len: Total number of tokens.
block_hashes: Block hashes for each block.
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
When ``block_size > hash_block_size`` consecutive hashes are merged
up to the group's ``block_size`` via ``BlockHashListWithBlockSize``.
mask_num: Number of tokens to skip from the beginning.
"""
if not block_hashes:
return
if not isinstance(block_hashes[0], str):
block_hashes = [
h.hex() # type: ignore[union-attr]
for h in block_hashes
]
for chunk_id, hash_val in enumerate(block_hashes):
if self.block_size == self.hash_block_size:
chunk_hashes: Iterable[BlockHash] = block_hashes
else:
chunk_hashes = BlockHashListWithBlockSize(
block_hashes, self.hash_block_size, self.block_size
)
for chunk_id, h in enumerate(chunk_hashes):
start_idx = chunk_id * self.block_size
if start_idx >= token_len:
break
end_idx = min(start_idx + self.block_size, token_len)
if start_idx < mask_num:
continue
else:
yield (
start_idx,
end_idx,
self._make_key_by_hash(
hash_val # type: ignore[arg-type]
),
)
yield start_idx, end_idx, self._make_key_by_hash(h.hex())
@dataclass
@@ -154,7 +165,7 @@ class RequestTracker:
req_id: str
token_len: int
allocated_block_ids: list[int]
allocated_block_ids: tuple[list[int], ...]
num_saved_tokens: int = 0
token_ids: list[int] | None = None
# Snapshot of the prefill range length at tracker creation time.
@@ -166,15 +177,18 @@ class RequestTracker:
self,
new_block_ids: tuple[list[int], ...] | list[int],
) -> None:
if len(new_block_ids) == 0:
new_block_ids = []
elif isinstance(new_block_ids, tuple):
new_block_ids = new_block_ids[0]
elif isinstance(new_block_ids, list):
pass
else:
raise ValueError(f"Unsupported new_block_ids type {type(new_block_ids)}")
self.allocated_block_ids.extend(new_block_ids)
# Backward-compat: accept a single list (broadcast to single group).
if isinstance(new_block_ids, list):
new_block_ids = (new_block_ids,)
if len(new_block_ids) != len(self.allocated_block_ids):
raise ValueError(
f"Group count mismatch: tracker has "
f"{len(self.allocated_block_ids)} groups, update has "
f"{len(new_block_ids)}"
)
for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True):
if new:
existing.extend(new)
@dataclass
@@ -183,7 +197,7 @@ class ReqMeta:
req_id: str
token_len_chunk: int
block_ids: list[int]
block_ids: tuple[list[int], ...]
block_hashes: list[BlockHash]
can_save: bool | None = None
@@ -22,7 +22,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
)
from vllm.logger import init_logger
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -45,7 +47,11 @@ def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
class MooncakeStoreScheduler:
"""Scheduler-side component for MooncakeStoreConnector."""
def __init__(self, vllm_config: VllmConfig):
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
):
assert vllm_config.kv_transfer_config is not None
self.kv_role = vllm_config.kv_transfer_config.kv_role
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
@@ -56,11 +62,11 @@ class MooncakeStoreScheduler:
self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size
self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size
self.original_block_size = vllm_config.cache_config.block_size
self._block_size = vllm_config.cache_config.block_size
if self.pcp_size > 1:
self._block_size *= self.pcp_size
if self.dcp_size > 1:
self._block_size *= self.dcp_size
# LCM for multi-group HMA; bs * pcp * dcp for single-group. Matches
# the engine's own scheduler block size by construction.
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self._discard_partial_chunks = (
vllm_config.kv_transfer_config.get_from_extra_config(
@@ -72,7 +78,7 @@ class MooncakeStoreScheduler:
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
self._preempted_req_ids: set[str] = set() # preempted requests
self._unfinished_requests: dict[str, tuple[Request, list[int]]] = {}
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
def get_num_new_matched_tokens(
@@ -126,9 +132,9 @@ class MooncakeStoreScheduler:
num_external_tokens: int,
):
"""Update state after block allocation."""
local_block_ids: list[int] = []
local_block_ids: tuple[list[int], ...] = ()
if num_external_tokens > 0:
local_block_ids = blocks.get_block_ids()[0]
local_block_ids = blocks.get_block_ids()
self._unfinished_requests[request.request_id] = (request, local_block_ids)
self._unfinished_request_ids.add(request.request_id)
@@ -189,11 +195,12 @@ class MooncakeStoreScheduler:
request_tuple = self._unfinished_requests.get(request.req_id)
request_real = request_tuple[0] # type: ignore[index]
if not isinstance(request.block_ids[0], list):
unfolded_block_ids = request.block_ids.copy()
if isinstance(request.block_ids, tuple):
# Multi-group: preserve per-group structure.
unfolded_block_ids = tuple(b.copy() for b in request.block_ids)
else:
# TODO: support HMA
unfolded_block_ids = request.block_ids[0].copy()
# Single-group legacy: list[int] -> 1-tuple.
unfolded_block_ids = (request.block_ids.copy(),)
prefill_tokens = _new_req_prefill_tokens(request)
request_tracker = RequestTracker(
@@ -237,9 +244,9 @@ class MooncakeStoreScheduler:
if req_id in self._preempted_req_ids:
# Resumed after preemption
if isinstance(new_block_ids, tuple):
block_ids_list = new_block_ids[0].copy()
new_block_ids = tuple(b.copy() for b in new_block_ids)
else:
block_ids_list = new_block_ids.copy()
new_block_ids = (new_block_ids.copy(),)
self._preempted_req_ids.discard(req_id)
load_spec = self.load_specs.pop(req_id, None)
request_tuple = self._unfinished_requests.get(req_id)
@@ -254,7 +261,7 @@ class MooncakeStoreScheduler:
request_tracker = RequestTracker(
req_id=req_id,
token_len=num_tokens_to_compute,
allocated_block_ids=block_ids_list,
allocated_block_ids=new_block_ids,
num_saved_tokens=0,
token_ids=prefill_tokens[:num_tokens_to_compute].copy(),
prefill_end_tokens=len(prefill_tokens),
@@ -361,7 +368,7 @@ class MooncakeStoreScheduler:
def request_finished(
self,
request: Request,
block_ids: list[int],
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
"""Determine whether to delay freeing blocks for async save."""
if self.kv_role == "kv_consumer":
@@ -370,11 +377,12 @@ class MooncakeStoreScheduler:
assert tracker is not None
if tracker.num_saved_tokens <= 0:
return False, None
delay_free_blocks = len(block_ids) > 0
total_blocks = sum(len(g) for g in block_ids)
delay_free_blocks = total_blocks > 0
if delay_free_blocks:
logger.debug(
"Delaying free of %d blocks for request %s",
len(block_ids),
total_blocks,
request.request_id,
)
return delay_free_blocks, None
@@ -10,6 +10,7 @@ Includes the store worker, transfer threads, lookup server,
and MooncakeDistributedStore integration.
"""
import dataclasses
import json
import os
import queue
@@ -36,15 +37,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
get_mooncake_dp_engine_index,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
ExternalCachedBlockPool,
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
ChunkedTokenDatabase,
KeyMetadata,
MooncakeStoreConnectorMetadata,
PoolKey,
ReqMeta,
)
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip, make_zmq_socket
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
from vllm.v1.core.kv_cache_utils import (
BlockHash,
maybe_convert_block_hash,
resolve_kv_cache_block_sizes,
)
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
logger = init_logger(__name__)
@@ -321,7 +332,7 @@ class KVTransferThread(threading.Thread):
def __init__(
self,
store: Any,
token_database: ChunkedTokenDatabase,
token_databases: list[ChunkedTokenDatabase],
block_size: int,
tp_rank: int,
ready_event: threading.Event,
@@ -332,7 +343,7 @@ class KVTransferThread(threading.Thread):
self.ready_event = ready_event
self.block_size = block_size
self.tp_rank = tp_rank
self.token_database = token_database
self.token_databases = token_databases
self.done_task_lock = threading.Lock()
self.request_queue: queue.Queue[Any] = queue.Queue()
self.finished_requests: set[str] = set()
@@ -385,7 +396,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
def __init__(
self,
store: Any,
token_database: ChunkedTokenDatabase,
coord: MooncakeStoreCoordinator,
token_databases: list[ChunkedTokenDatabase],
block_size: int,
tp_rank: int,
put_step: int,
@@ -396,13 +408,14 @@ class KVCacheStoreSendingThread(KVTransferThread):
):
super().__init__(
store,
token_database,
token_databases,
block_size,
tp_rank,
ready_event,
name="KVCacheStoreSendingThread",
)
self.put_step = put_step
self.coord = coord
self.kv_role = kv_role
self.stored_requests: defaultdict[str, int] = defaultdict(int)
self.enable_kv_event = enable_kv_event
@@ -449,14 +462,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
return True
def _handle_request(self, req_meta: ReqMeta):
token_len = req_meta.token_len_chunk
block_ids = req_meta.block_ids
# Cache hits are always a multiple of ``lcm_block_size`` tokens
lcm_block_size = self.coord.lcm_block_size
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
block_ids_per_group = req_meta.block_ids
req_id = req_meta.req_id
current_event = req_meta.current_event
if req_id not in self.stored_requests:
self.request_queue.task_done()
return
if token_len == 0:
self.dec_stored_request(req_id)
self.request_queue.task_done()
return
if self._should_skip_request(req_id):
logger.debug(
"Skipping Mooncake store for request %s while CPU/disk offloading "
@@ -467,23 +486,34 @@ class KVCacheStoreSendingThread(KVTransferThread):
self.request_queue.task_done()
return
starts = []
ends = []
keys = []
# Within each lcm region only per-spec relevant chunks are loaded
# (e.g., SWA or linear attn), so mask out irrelevant chunks
store_masks = self.coord.store_mask(token_len)
starts: list[int] = []
ends: list[int] = []
keys: list[str] = []
block_hashes: list[BlockHash] = []
for index, (start, end, key) in enumerate(
self.token_database.process_tokens(token_len, req_meta.block_hashes)
):
starts.append(start)
ends.append(end)
keys.append(key.to_string())
block_hashes.append(req_meta.block_hashes[index])
group_indices: list[int] = []
for g_idx, db in enumerate(self.token_databases):
mask = store_masks[g_idx]
for chunk_idx, (start, end, key) in enumerate(
db.process_tokens(token_len, req_meta.block_hashes)
):
if chunk_idx >= len(mask) or not mask[chunk_idx]:
continue
starts.append(start)
ends.append(end)
keys.append(key.to_string())
block_hashes.append(req_meta.block_hashes[chunk_idx])
group_indices.append(g_idx)
# Apply put_step striding for TP
starts = starts[self.tp_rank % self.put_step :: self.put_step]
ends = ends[self.tp_rank % self.put_step :: self.put_step]
keys = keys[self.tp_rank % self.put_step :: self.put_step]
block_hashes = block_hashes[self.tp_rank % self.put_step :: self.put_step]
sl = slice(self.tp_rank % self.put_step, None, self.put_step)
starts = starts[sl]
ends = ends[sl]
keys = keys[sl]
block_hashes = block_hashes[sl]
group_indices = group_indices[sl]
if not keys:
self.dec_stored_request(req_id)
@@ -501,38 +531,37 @@ class KVCacheStoreSendingThread(KVTransferThread):
ends = [ends[i] for i in missing_indices]
keys = [keys[i] for i in missing_indices]
block_hashes = [block_hashes[i] for i in missing_indices]
group_indices = [group_indices[i] for i in missing_indices]
logger.debug(
"Storing KV cache for %d out of %d blocks "
"(missing_count=%d) for request %s",
"Storing KV cache for %d blocks (groups=%s) for request %s",
len(keys),
token_len // self.block_size,
len(missing_indices),
set(group_indices),
req_id,
)
addrs = []
sizes = []
addrs: list[list[int]] = []
sizes: list[list[int]] = []
stored_events: list[BlockStored] = []
prev_key = None
# parent_block_hash chains live within a group, not across.
prev_key_per_group: dict[int, Any] = {}
new_block_hashes = [maybe_convert_block_hash(bh) for bh in block_hashes]
for index, start in enumerate(starts):
addr, size, _ = self.token_database.prepare_value(
start, ends[index], block_ids
)
for idx, (s, e, g_idx) in enumerate(
zip(starts, ends, group_indices, strict=True)
):
db = self.token_databases[g_idx]
addr, size, _ = db.prepare_value(s, e, block_ids_per_group[g_idx])
addrs.append(addr)
sizes.append(size)
if self.enable_kv_event:
token_ids = (
req_meta.token_ids[start : ends[index]]
if req_meta.token_ids is not None
else None
req_meta.token_ids[s:e] if req_meta.token_ids is not None else None
)
stored_event = BlockStored(
block_hashes=[new_block_hashes[index]],
parent_block_hash=prev_key,
block_hashes=[new_block_hashes[idx]],
parent_block_hash=prev_key_per_group.get(g_idx),
token_ids=token_ids,
block_size=req_meta.original_block_size,
lora_id=None,
@@ -540,7 +569,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
lora_name=None,
)
stored_events.append(stored_event)
prev_key = new_block_hashes[index]
prev_key_per_group[g_idx] = new_block_hashes[idx]
if current_event is not None:
current_event.synchronize()
@@ -600,7 +629,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
def __init__(
self,
store: Any,
token_database: ChunkedTokenDatabase,
coord: MooncakeStoreCoordinator,
token_databases: list[ChunkedTokenDatabase],
block_size: int,
tp_rank: int,
ready_event: threading.Event,
@@ -608,7 +638,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
):
super().__init__(
store,
token_database,
token_databases,
block_size,
tp_rank,
ready_event,
@@ -622,6 +652,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
disk_offload_buffer_budget_bytes
)
)
self.coord = coord
def _handle_request(self, req_meta: ReqMeta):
token_len = req_meta.load_spec.token_len # type: ignore[union-attr]
@@ -632,32 +663,31 @@ class KVCacheStoreRecvingThread(KVTransferThread):
* self.block_size
)
addr_list = []
size_list = []
key_list = []
for start, end, key in self.token_database.process_tokens(
token_len, req_meta.block_hashes, mask_num
):
addr, size, _ = self.token_database.prepare_value(
start, end, req_meta.block_ids
)
key_list.append(key.to_string())
addr_list.append(addr)
size_list.append(size)
# Skip chunks the consumer's per-group spec wouldn't populate
# locally (e.g. SWA pre-window) even if the producer stored them.
load_mask_per_group = self.coord.load_mask(req_meta.block_hashes, token_len)
addr_list: list[list[int]] = []
size_list: list[list[int]] = []
key_list: list[str] = []
for g_idx, db in enumerate(self.token_databases):
mask = load_mask_per_group[g_idx]
for start, end, key in db.process_tokens(
token_len, req_meta.block_hashes, mask_num
):
chunk_idx = start // db.block_size
if chunk_idx >= len(mask) or not mask[chunk_idx]:
continue
addr, size, _ = db.prepare_value(start, end, req_meta.block_ids[g_idx])
key_list.append(key.to_string())
addr_list.append(addr)
size_list.append(size)
# Rotate lists by tp_rank for load balancing
key_list_c = (
key_list[self.tp_rank % len(key_list) :]
+ key_list[: self.tp_rank % len(key_list)]
)
addr_list_c = (
addr_list[self.tp_rank % len(addr_list) :]
+ addr_list[: self.tp_rank % len(addr_list)]
)
size_list_c = (
size_list[self.tp_rank % len(size_list) :]
+ size_list[: self.tp_rank % len(size_list)]
)
rotation = self.tp_rank % len(key_list)
key_list_c = key_list[rotation:] + key_list[:rotation]
addr_list_c = addr_list[rotation:] + addr_list[:rotation]
size_list_c = size_list[rotation:] + size_list[:rotation]
load_batches = [(key_list_c, addr_list_c, size_list_c)]
if self.usable_disk_offload_buffer_budget_bytes is not None:
@@ -737,7 +767,11 @@ class KVCacheStoreRecvingThread(KVTransferThread):
class MooncakeStoreWorker:
"""Worker-side component for MooncakeStoreConnector."""
def __init__(self, vllm_config: VllmConfig):
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
):
try:
from mooncake.store import ( # type: ignore
MooncakeDistributedStore,
@@ -771,11 +805,9 @@ class MooncakeStoreWorker:
)
self.cache_config = vllm_config.cache_config
self.original_block_size = self.cache_config.block_size
self.block_size = self.cache_config.block_size
if self.pcp_size > 1:
self.block_size *= self.pcp_size
if self.dcp_size > 1:
self.block_size *= self.dcp_size
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.num_layers = model_config.get_num_layers(parallel_config)
self.use_mla = False
@@ -806,8 +838,6 @@ class MooncakeStoreWorker:
pp_rank=self.pp_rank,
)
self.token_database = ChunkedTokenDatabase(self.metadata, self.block_size)
# Initialize MooncakeDistributedStore with its own TransferEngine
store_config = MooncakeStoreConfig.load_from_env()
extra_config = (
@@ -892,6 +922,45 @@ class MooncakeStoreWorker:
self.kv_recv_thread: KVCacheStoreRecvingThread | None = None
self.finished_store_req: set[str] = set()
self._kv_cache_config = kv_cache_config
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
# self.block_size (= scheduler_block_size) so the coordinator's
# ``block_size % hash_block_size == 0`` invariant holds.
groups = list(kv_cache_config.kv_cache_groups)
if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size:
g = groups[0]
groups = [
dataclasses.replace(
g,
kv_cache_spec=dataclasses.replace(
g.kv_cache_spec, block_size=self.block_size
),
)
]
self._kv_cache_groups: list[KVCacheGroupSpec] = groups
spec_cfg = getattr(vllm_config, "speculative_config", None)
use_eagle = bool(
spec_cfg.use_eagle()
if spec_cfg is not None and callable(getattr(spec_cfg, "use_eagle", None))
else False
)
self.coord = MooncakeStoreCoordinator(
self._kv_cache_groups,
scheduler_block_size=self.block_size,
hash_block_size=self.hash_block_size,
use_eagle=use_eagle,
)
# One ChunkedTokenDatabase per group; addresses populated in
# register_kv_caches once the kv-cache layout is known.
self.token_dbs: list[ChunkedTokenDatabase] = [
ChunkedTokenDatabase(
dataclasses.replace(self.metadata, group_id=g_idx),
g.kv_cache_spec.block_size,
hash_block_size=self.hash_block_size,
)
for g_idx, g in enumerate(self._kv_cache_groups)
]
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
"""Register a cross-layers KV cache tensor.
@@ -901,93 +970,86 @@ class MooncakeStoreWorker:
"""
self.register_kv_caches({"__cross_layer__": kv_cache})
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
def register_kv_caches(
self,
kv_caches: dict[str, torch.Tensor | list[torch.Tensor]],
) -> None:
"""Register KV cache tensors and start transfer threads."""
# TODO(yifan): we haven't supported HMA yet.
first_kv_cache = next(iter(kv_caches.values()))
if not kv_caches:
logger.warning("No KV caches to offload.")
return
# Resolve each entry to a representative tensor for storage
# deduplication. For attention layers the value is already a tensor;
# for Mamba layers it is a list of tensors that all share the same
# underlying raw storage, so we take the first one.
def _repr_tensor(v: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
assert isinstance(v, torch.Tensor | list)
return v if isinstance(v, torch.Tensor) else v[0]
# num_blocks from cache_config is authoritative (set after
# profiling, before KV cache allocation).
assert self.cache_config.num_gpu_blocks is not None
self.num_blocks = self.cache_config.num_gpu_blocks
# Detect the KV cache memory layout using the stride-based
# approach from simple_kv_offload/worker.py.
#
# The physical layout varies across attention backends:
# FlashAttn/ROCm : (2, num_blocks, ...) → K/V outermost
# FlashInfer/MLA : (num_blocks, ...) → blocks outermost
#
# We derive page_size_bytes = storage.nbytes() // num_blocks,
# then classify dims: any dim whose byte-stride exceeds
# page_size_bytes must be an outer segment dim (e.g. the K/V
# dim of size 2). For those backends we register each segment
# (K, V) as a separate base-address so that the per-block
# offset arithmetic in prepare_value() stays correct.
storage = first_kv_cache.untyped_storage()
el = first_kv_cache.element_size()
page_size_bytes = storage.nbytes() // self.num_blocks
outer_dims = [
d
for d in range(first_kv_cache.ndim)
if first_kv_cache.stride(d) * el > page_size_bytes
]
# Register buffers with the store (deduplicate shared storages)
# and record per-segment base addresses for every layer.
seen_ptrs: set[int] = set()
self.kv_caches_base_addr: list[int] = []
self.block_len: list[int] = []
addrs: list[int] = []
block_lens: list[int] = []
for cache in kv_caches.values():
for value in kv_caches.values():
cache = _repr_tensor(value)
cache_storage = cache.untyped_storage()
base_addr = cache_storage.data_ptr()
if base_addr in seen_ptrs:
continue
seen_ptrs.add(base_addr)
region_len = cache_storage.nbytes()
if base_addr not in seen_ptrs:
seen_ptrs.add(base_addr)
ret = self.store.register_buffer(base_addr, region_len)
if ret != 0:
logger.error(
"register_buffer failed for addr %#x len %d: %d",
base_addr,
region_len,
ret,
)
ret = self.store.register_buffer(base_addr, region_len)
if ret != 0:
logger.error(
"register_buffer failed for addr %#x len %d: %d",
base_addr,
region_len,
ret,
)
# Detect layout via stride: a dim whose byte-stride exceeds
# page_size_bytes is an outer segment dim (e.g. the K/V dim of
# FlashAttn's (2, num_blocks, ...)). FlashInfer/MLA's blocks-
# outermost layout has no such dim and yields a single segment.
el = cache.element_size()
page_size_bytes = region_len // self.num_blocks
outer_dims = [
d for d in range(cache.ndim) if cache.stride(d) * el > page_size_bytes
]
if not outer_dims:
# Blocks-first layout (FlashInfer / MLA): one segment.
self.kv_caches_base_addr.append(base_addr)
self.block_len.append(page_size_bytes)
addrs.append(base_addr)
block_lens.append(page_size_bytes)
else:
# K/V-first layout (FlashAttn / ROCm): split segments.
seg_stride = cache.stride(outer_dims[0]) * el
for idx in range(cache.shape[outer_dims[0]]):
self.kv_caches_base_addr.append(base_addr + idx * seg_stride)
self.block_len.append(seg_stride // self.num_blocks)
addrs.append(base_addr + idx * seg_stride)
block_lens.append(seg_stride // self.num_blocks)
logger.info(
"Registering KV_Caches. use_mla: %s, shape %s, "
"num_blocks: %d, block_len: %s, "
"per_key_bytes: %d, "
"num_segments: %d",
self.use_mla,
first_kv_cache.shape,
"Registered KV caches: num_groups=%d, num_segments=%d, num_blocks=%d",
len(self.token_dbs),
len(addrs),
self.num_blocks,
list(set(self.block_len)),
sum(self.block_len),
len(self.kv_caches_base_addr),
)
self.token_database.set_kv_caches_base_addr(self.kv_caches_base_addr)
self.token_database.set_block_len(self.block_len)
for db in self.token_dbs:
db.set_kv_caches_base_addr(addrs)
db.set_block_len(block_lens)
# Start transfer threads
if self.kv_role in ["kv_producer", "kv_both"]:
ready_event_sending = threading.Event()
self.kv_send_thread = KVCacheStoreSendingThread(
self.store,
self.token_database,
self.coord,
self.token_dbs,
self.block_size,
self.tp_rank,
self.put_step,
@@ -1001,7 +1063,8 @@ class MooncakeStoreWorker:
ready_event_recving = threading.Event()
self.kv_recv_thread = KVCacheStoreRecvingThread(
self.store,
self.token_database,
self.coord,
self.token_dbs,
self.block_size,
self.tp_rank,
ready_event_recving,
@@ -1122,62 +1185,55 @@ class MooncakeStoreWorker:
return finished_sending
def lookup(
self,
token_len: int,
block_hashes: list[BlockHash],
) -> int:
def lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
"""Check how many prefix tokens exist in the store.
Checks across all TP ranks and PP ranks.
"""
end = 0
keys: list[str] = []
if not block_hashes or token_len <= 0:
return 0
# Build per-(group, hash) candidate keys expanded across TP/PP.
# candidate_meta[i] is the (group_id, hash_bytes) for candidate_keys[i].
candidate_keys: list[str] = []
candidate_meta: list[tuple[int, bytes]] = []
tp_count = min(self.tp_size, self.num_kv_head)
for g_idx, db in enumerate(self.token_dbs):
spec_block_size = db.block_size
group_hashes = self.coord.block_hashes_for_spec(
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
)
for chunk_id, h in enumerate(group_hashes):
start_idx = chunk_id * spec_block_size
if start_idx >= token_len:
break
for tp in range(tp_count):
for pp in range(self.pp_size):
md = dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp)
candidate_keys.append(PoolKey(md, h.hex()).to_string())
candidate_meta.append((g_idx, bytes(h)))
if not candidate_keys:
return 0
try:
starts: list[int] = []
for start, end, key in self.token_database.process_tokens(
token_len, block_hashes
):
keys.append(key.to_string())
starts.append(start)
# Expand keys for all TP ranks
multi_tp_keys = keys[:]
for i in range(1, min(self.tp_size, self.num_kv_head)):
for item in keys:
new_str = item.replace("@tp_rank:0", f"@tp_rank:{i}", 1)
multi_tp_keys.append(new_str)
# Expand keys for all PP ranks
pp_base_keys = multi_tp_keys.copy()
for i in range(1, self.pp_size):
for item in pp_base_keys:
new_str = item.replace("@pp_rank:0", f"@pp_rank:{i}", 1)
multi_tp_keys.append(new_str)
res = self.store.batch_is_exist(multi_tp_keys)
num_block = len(keys)
multi_tp_values = [
res[i * num_block : (i + 1) * num_block]
for i in range(min(self.tp_size, self.num_kv_head) * self.pp_size)
]
index = self._find_min_first_non_one_index(multi_tp_values)
if index != -1:
return starts[index]
res = self.store.batch_is_exist(candidate_keys)
except Exception as e:
logger.error("Remote connection failed in lookup: %s", e)
return 0
return end
@staticmethod
def _find_min_first_non_one_index(
arr: list[list[int]],
) -> int:
try:
return min(idx for row in arr for idx, val in enumerate(row) if val != 1)
except ValueError:
return -1
# A (group, hash) is "present" only when every TP*PP rank has it.
expected_per_key = max(1, tp_count * self.pp_size)
present_count: dict[tuple[int, bytes], int] = {}
for gh, exists in zip(candidate_meta, res, strict=True):
if exists == 1:
present_count[gh] = present_count.get(gh, 0) + 1
exists_set = {gh for gh, c in present_count.items() if c >= expected_per_key}
_masks, hit_length = self.coord.find_longest_cache_hit(
block_hashes, token_len, ExternalCachedBlockPool(exists_set)
)
return hit_length
def get_kv_events(self) -> list[BlockStored]:
if self.enable_kv_events and self.kv_send_thread is not None:
@@ -1220,7 +1276,8 @@ class LookupKeyServer:
token_len = int.from_bytes(all_frames[0], byteorder="big")
hash_frames = all_frames[1:]
hashes_str = self.decoder.decode(hash_frames)
result = self.store_worker.lookup(token_len, hashes_str)
block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str]
result = self.store_worker.lookup(token_len, block_hashes)
response = result.to_bytes(4, "big")
self.socket.send(response)