[KVConnector][DSV4] HMA support for Mooncake store connector (#42828)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
@@ -76,7 +77,7 @@ class MooncakeStoreKVEvents(KVConnectorKVEvents):
|
||||
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
|
||||
|
||||
|
||||
class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""KV connector using MooncakeDistributedStore as shared KV pool."""
|
||||
|
||||
@property
|
||||
@@ -87,6 +88,36 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
== "true"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_kv_cache_config(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> None:
|
||||
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
|
||||
|
||||
unsupported: list[str] = []
|
||||
cache_block_size = vllm_config.cache_config.block_size
|
||||
for g_idx, g in enumerate(kv_cache_config.kv_cache_groups):
|
||||
spec = g.kv_cache_spec
|
||||
if isinstance(spec, CrossAttentionSpec):
|
||||
unsupported.append(f"group {g_idx}: CrossAttentionSpec")
|
||||
# Enforce Mamba align mode
|
||||
if isinstance(spec, MambaSpec) and spec.block_size != cache_block_size:
|
||||
unsupported.append(
|
||||
f"group {g_idx}: MambaSpec with block_size="
|
||||
f"{spec.block_size} != cache_config.block_size="
|
||||
f"{cache_block_size} (mamba_cache_mode != 'align')"
|
||||
)
|
||||
pcp = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
dcp = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1:
|
||||
unsupported.append(
|
||||
f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention"
|
||||
)
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -99,6 +130,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
kv_cache_config=kv_cache_config, # type: ignore[arg-type]
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
assert kv_cache_config is not None, "kv_cache_config is required"
|
||||
self._validate_kv_cache_config(vllm_config, kv_cache_config)
|
||||
self._kv_cache_config = kv_cache_config
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
self._kv_cache_events: MooncakeStoreKVEvents | None = None
|
||||
|
||||
@@ -106,9 +140,11 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
self.connector_worker: MooncakeStoreWorker | None = None
|
||||
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = MooncakeStoreScheduler(vllm_config)
|
||||
self.connector_scheduler = MooncakeStoreScheduler(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
else:
|
||||
self.connector_worker = MooncakeStoreWorker(vllm_config)
|
||||
self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config)
|
||||
|
||||
# ============================================================
|
||||
# Scheduler-side methods
|
||||
@@ -146,6 +182,13 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
return self.request_finished_all_groups(request, (block_ids,))
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
@@ -184,6 +227,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
self, kv_cache: torch.Tensor, attn_backend: type
|
||||
):
|
||||
assert self.connector_worker is not None
|
||||
assert (
|
||||
self._kv_cache_config is not None
|
||||
and len(self._kv_cache_config.kv_cache_groups) == 1
|
||||
), "Cross-layer KV cache does not supported with hybrid models"
|
||||
self.connector_worker.register_cross_layers_kv_caches(kv_cache)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
|
||||
|
||||
from typing import cast
|
||||
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashList,
|
||||
BlockHashListWithBlockSize,
|
||||
KVCacheBlock,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
SingleTypeKVCacheManager,
|
||||
spec_manager_map,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
# Dummy placeholder hash for store_mask's template computation.
|
||||
_DUMMY_BLOCK_HASH = BlockHash(b"\x00" * 32)
|
||||
|
||||
|
||||
class ExternalCachedBlockPool:
|
||||
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
|
||||
|
||||
def __init__(self, exists: set[tuple[int, bytes]] | None = None) -> None:
|
||||
# ``exists=None`` is used on the recv side where hit_length is already
|
||||
# determined and we just want each spec's manager to apply its own mask.
|
||||
self._exists = exists
|
||||
self.null_block = KVCacheBlock(block_id=0)
|
||||
# Dummy ID 1 for present block for duck-typing.
|
||||
self._present_block = KVCacheBlock(block_id=1)
|
||||
|
||||
def get_cached_block(
|
||||
self,
|
||||
block_hash: BlockHash,
|
||||
group_ids: list[int],
|
||||
) -> list[KVCacheBlock] | None:
|
||||
# Mirrors BlockPool.get_cached_block: hit only when every group_id
|
||||
# (groups sharing a spec) has the hash cached.
|
||||
if self._exists is None:
|
||||
return [self._present_block] * len(group_ids)
|
||||
h = bytes(block_hash)
|
||||
if all((g, h) in self._exists for g in group_ids):
|
||||
return [self._present_block] * len(group_ids)
|
||||
return None
|
||||
|
||||
|
||||
class MooncakeStoreCoordinator:
|
||||
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
|
||||
``ExternalCachedBlockPool``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
scheduler_block_size: int,
|
||||
hash_block_size: int,
|
||||
use_eagle: bool = False,
|
||||
) -> None:
|
||||
assert all(
|
||||
g.kv_cache_spec.block_size % hash_block_size == 0 for g in kv_cache_groups
|
||||
), "block_size must be divisible by hash_block_size"
|
||||
assert scheduler_block_size % hash_block_size == 0, (
|
||||
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
|
||||
f"hash_block_size ({hash_block_size})"
|
||||
)
|
||||
assert all(
|
||||
scheduler_block_size % g.kv_cache_spec.block_size == 0
|
||||
for g in kv_cache_groups
|
||||
), "scheduler_block_size must be a multiple of each group's block_size"
|
||||
self.kv_cache_groups = kv_cache_groups
|
||||
self.hash_block_size = hash_block_size
|
||||
self.lcm_block_size = scheduler_block_size
|
||||
self.use_eagle = use_eagle
|
||||
self._verify_and_split_kv_cache_groups()
|
||||
|
||||
def _verify_and_split_kv_cache_groups(self) -> None:
|
||||
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
"""
|
||||
attention_groups: list[
|
||||
tuple[KVCacheSpec, list[int], type[SingleTypeKVCacheManager]]
|
||||
] = []
|
||||
for i, g in enumerate(self.kv_cache_groups):
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
manager_cls = spec_manager_map[type(spec)]
|
||||
for existing_spec, group_ids, existing_cls in attention_groups:
|
||||
if existing_spec == spec:
|
||||
assert manager_cls is existing_cls
|
||||
group_ids.append(i)
|
||||
break
|
||||
else:
|
||||
attention_groups.append((spec, [i], manager_cls))
|
||||
# Full attention first (matches upstream convergence ordering).
|
||||
self.attention_groups = sorted(
|
||||
attention_groups,
|
||||
key=lambda x: not isinstance(x[0], FullAttentionSpec),
|
||||
)
|
||||
self.eagle_attn_group_indices: set[int] = {
|
||||
i
|
||||
for i, (_, group_ids, _) in enumerate(self.attention_groups)
|
||||
if any(self.kv_cache_groups[gid].is_eagle_group for gid in group_ids)
|
||||
}
|
||||
if self.use_eagle and not self.eagle_attn_group_indices:
|
||||
self.eagle_attn_group_indices = set(range(len(self.attention_groups)))
|
||||
|
||||
def find_longest_cache_hit(
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
max_length: int,
|
||||
cached_block_pool: ExternalCachedBlockPool,
|
||||
) -> tuple[tuple[list[bool], ...], int]:
|
||||
"""Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff
|
||||
group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only);
|
||||
recv-side callers skip False slots."""
|
||||
blocks_per_group, hit_length = self._find_hit_blocks(
|
||||
block_hashes, max_length, cached_block_pool
|
||||
)
|
||||
masks = tuple(
|
||||
[blk is not cached_block_pool.null_block for blk in blocks]
|
||||
for blocks in blocks_per_group
|
||||
)
|
||||
return masks, hit_length
|
||||
|
||||
def load_mask(
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
token_len: int,
|
||||
) -> tuple[list[bool], ...]:
|
||||
"""Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s
|
||||
spec would populate chunk ``i`` locally at length ``token_len``
|
||||
(e.g. SWA / Mamba tail-only).
|
||||
"""
|
||||
masks, _ = self.find_longest_cache_hit(
|
||||
block_hashes, token_len, ExternalCachedBlockPool()
|
||||
)
|
||||
return masks
|
||||
|
||||
def store_mask(self, aligned_token_len: int) -> tuple[list[bool], ...]:
|
||||
"""Per-group store masks: ``mask[g][i]`` is True iff chunk ``i`` of
|
||||
group ``g`` would be populated by some future cache hit at length
|
||||
``L = N * lcm_block_size <= aligned_token_len``.
|
||||
"""
|
||||
assert aligned_token_len % self.lcm_block_size == 0, (
|
||||
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
|
||||
f"lcm_block_size ({self.lcm_block_size})"
|
||||
)
|
||||
if aligned_token_len == 0:
|
||||
return tuple([] for _ in self.kv_cache_groups)
|
||||
|
||||
num_chunks_per_group = [
|
||||
aligned_token_len // g.kv_cache_spec.block_size
|
||||
for g in self.kv_cache_groups
|
||||
]
|
||||
|
||||
# Fast path: single group or full attn groups or uniform block_sizes
|
||||
if all(
|
||||
isinstance(spec, FullAttentionSpec)
|
||||
or spec.block_size == self.lcm_block_size
|
||||
for spec, _, _ in self.attention_groups
|
||||
):
|
||||
return tuple([True] * n for n in num_chunks_per_group)
|
||||
|
||||
n_segments = aligned_token_len // self.lcm_block_size
|
||||
dummy_hashes: list[BlockHash] = [_DUMMY_BLOCK_HASH] * (
|
||||
self.lcm_block_size // self.hash_block_size
|
||||
)
|
||||
template_masks, _ = self.find_longest_cache_hit(
|
||||
dummy_hashes,
|
||||
max_length=self.lcm_block_size,
|
||||
cached_block_pool=ExternalCachedBlockPool(),
|
||||
)
|
||||
return tuple(
|
||||
list(template_masks[g]) * n_segments
|
||||
for g in range(len(self.kv_cache_groups))
|
||||
)
|
||||
|
||||
def block_hashes_for_spec(
|
||||
self, block_hashes: list[BlockHash], spec: KVCacheSpec
|
||||
) -> BlockHashList:
|
||||
if spec.block_size == self.hash_block_size:
|
||||
return block_hashes
|
||||
return BlockHashListWithBlockSize(
|
||||
block_hashes, self.hash_block_size, spec.block_size
|
||||
)
|
||||
|
||||
def _find_hit_blocks(
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
max_length: int,
|
||||
cached_block_pool: ExternalCachedBlockPool,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
"""Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
"""
|
||||
if len(self.attention_groups) == 1:
|
||||
spec, group_ids, manager_cls = self.attention_groups[0]
|
||||
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
||||
hit_blocks = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=hashes,
|
||||
max_length=max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=(0 in self.eagle_attn_group_indices),
|
||||
alignment_tokens=spec.block_size,
|
||||
)
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
||||
for gid, blks in zip(group_ids, hit_blocks, strict=True):
|
||||
blocks_by_group[gid] = blks
|
||||
return tuple(blocks_by_group), len(hit_blocks[0]) * spec.block_size
|
||||
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
hit_length = max_length
|
||||
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
|
||||
|
||||
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
|
||||
self.attention_groups[0][0], FullAttentionSpec
|
||||
)
|
||||
eagle_verified: set[int] = set()
|
||||
|
||||
while True:
|
||||
curr_hit_length = hit_length
|
||||
|
||||
for idx, (spec, group_ids, manager_cls) in enumerate(self.attention_groups):
|
||||
cached = hit_blocks_by_group[group_ids[0]]
|
||||
if isinstance(spec, FullAttentionSpec) and cached is not None:
|
||||
curr_hit_length = (
|
||||
curr_hit_length // spec.block_size * spec.block_size
|
||||
)
|
||||
continue
|
||||
|
||||
use_eagle = (
|
||||
idx in self.eagle_attn_group_indices and idx not in eagle_verified
|
||||
)
|
||||
_max_length = curr_hit_length
|
||||
if use_eagle:
|
||||
_max_length = min(curr_hit_length + spec.block_size, max_length)
|
||||
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
||||
hit_blocks = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=hashes,
|
||||
max_length=_max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=use_eagle,
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
)
|
||||
_new_hit_length = len(hit_blocks[0]) * spec.block_size
|
||||
if use_eagle:
|
||||
eagle_verified.add(idx)
|
||||
elif _new_hit_length < curr_hit_length:
|
||||
eagle_verified.clear()
|
||||
curr_hit_length = _new_hit_length
|
||||
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
|
||||
hit_blocks_by_group[gid] = blocks
|
||||
|
||||
if curr_hit_length >= hit_length:
|
||||
break
|
||||
hit_length = curr_hit_length
|
||||
if is_simple_hybrid:
|
||||
break
|
||||
|
||||
# Truncate full-attention hit_blocks to final converged length;
|
||||
# other specs already trim themselves inside their hit logic.
|
||||
spec0, group_ids0, _ = self.attention_groups[0]
|
||||
if isinstance(spec0, FullAttentionSpec):
|
||||
num_blocks = hit_length // spec0.block_size
|
||||
for gid in group_ids0:
|
||||
full_blks = hit_blocks_by_group[gid]
|
||||
assert full_blks is not None
|
||||
del full_blks[num_blocks:]
|
||||
|
||||
return (
|
||||
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
|
||||
hit_length,
|
||||
)
|
||||
|
||||
|
||||
def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return next(iter(spec.kv_cache_specs.values()))
|
||||
return spec
|
||||
@@ -15,7 +15,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashListWithBlockSize,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -29,6 +32,7 @@ class KeyMetadata:
|
||||
pcp_rank: int
|
||||
dcp_rank: int
|
||||
pp_rank: int
|
||||
group_id: int = 0
|
||||
|
||||
|
||||
@dataclass(order=True)
|
||||
@@ -46,6 +50,7 @@ class PoolKey:
|
||||
self.key_metadata.pcp_rank,
|
||||
self.key_metadata.dcp_rank,
|
||||
self.key_metadata.pp_rank,
|
||||
self.key_metadata.group_id,
|
||||
self.chunk_hash,
|
||||
)
|
||||
)
|
||||
@@ -57,6 +62,7 @@ class PoolKey:
|
||||
f"@pcp{self.key_metadata.pcp_rank}"
|
||||
f"@dcp{self.key_metadata.dcp_rank}"
|
||||
f"@pp_rank:{self.key_metadata.pp_rank}"
|
||||
f"@group:{self.key_metadata.group_id}"
|
||||
f"@{self.chunk_hash}"
|
||||
)
|
||||
|
||||
@@ -64,9 +70,20 @@ class PoolKey:
|
||||
class ChunkedTokenDatabase:
|
||||
"""Maps token positions to store keys and GPU memory addresses."""
|
||||
|
||||
def __init__(self, metadata: KeyMetadata, block_size: int):
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int | None = None,
|
||||
):
|
||||
self.metadata = metadata
|
||||
self.block_size = block_size
|
||||
self.hash_block_size = hash_block_size or block_size
|
||||
if self.block_size % self.hash_block_size != 0:
|
||||
raise ValueError(
|
||||
f"block_size ({self.block_size}) must be a multiple of "
|
||||
f"hash_block_size ({self.hash_block_size})"
|
||||
)
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
|
||||
@@ -77,9 +94,6 @@ class ChunkedTokenDatabase:
|
||||
self.kv_caches_base_addr = kv_caches_base_addr
|
||||
|
||||
def set_block_len(self, block_len: list[int]):
|
||||
for length in block_len:
|
||||
if length % self.block_size != 0:
|
||||
raise ValueError(f"block_len {length} % {self.block_size} != 0")
|
||||
self.block_len = block_len
|
||||
|
||||
def prepare_value(
|
||||
@@ -96,7 +110,8 @@ class ChunkedTokenDatabase:
|
||||
length = len(self.block_len)
|
||||
for index, base_addr in enumerate(self.kv_caches_base_addr):
|
||||
addr = base_addr + block_id * self.block_len[index % length]
|
||||
size = self.block_len[index % length] // self.block_size * (end - start)
|
||||
assert (end - start) % self.block_size == 0
|
||||
size = self.block_len[index % length] * cdiv(end - start, self.block_size)
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
return addr_list, size_list, block_id
|
||||
@@ -104,38 +119,34 @@ class ChunkedTokenDatabase:
|
||||
def process_tokens(
|
||||
self,
|
||||
token_len: int,
|
||||
block_hashes: list[BlockHash] | list[str],
|
||||
block_hashes: list[BlockHash],
|
||||
mask_num: int = 0,
|
||||
) -> Iterable[tuple[int, int, PoolKey]]:
|
||||
"""Process tokens and yield (start_idx, end_idx, pool_key) tuples.
|
||||
|
||||
Args:
|
||||
token_len: Total number of tokens.
|
||||
block_hashes: Block hashes for each block.
|
||||
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
|
||||
When ``block_size > hash_block_size`` consecutive hashes are merged
|
||||
up to the group's ``block_size`` via ``BlockHashListWithBlockSize``.
|
||||
mask_num: Number of tokens to skip from the beginning.
|
||||
"""
|
||||
if not block_hashes:
|
||||
return
|
||||
if not isinstance(block_hashes[0], str):
|
||||
block_hashes = [
|
||||
h.hex() # type: ignore[union-attr]
|
||||
for h in block_hashes
|
||||
]
|
||||
for chunk_id, hash_val in enumerate(block_hashes):
|
||||
if self.block_size == self.hash_block_size:
|
||||
chunk_hashes: Iterable[BlockHash] = block_hashes
|
||||
else:
|
||||
chunk_hashes = BlockHashListWithBlockSize(
|
||||
block_hashes, self.hash_block_size, self.block_size
|
||||
)
|
||||
for chunk_id, h in enumerate(chunk_hashes):
|
||||
start_idx = chunk_id * self.block_size
|
||||
if start_idx >= token_len:
|
||||
break
|
||||
end_idx = min(start_idx + self.block_size, token_len)
|
||||
if start_idx < mask_num:
|
||||
continue
|
||||
else:
|
||||
yield (
|
||||
start_idx,
|
||||
end_idx,
|
||||
self._make_key_by_hash(
|
||||
hash_val # type: ignore[arg-type]
|
||||
),
|
||||
)
|
||||
yield start_idx, end_idx, self._make_key_by_hash(h.hex())
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -154,7 +165,7 @@ class RequestTracker:
|
||||
|
||||
req_id: str
|
||||
token_len: int
|
||||
allocated_block_ids: list[int]
|
||||
allocated_block_ids: tuple[list[int], ...]
|
||||
num_saved_tokens: int = 0
|
||||
token_ids: list[int] | None = None
|
||||
# Snapshot of the prefill range length at tracker creation time.
|
||||
@@ -166,15 +177,18 @@ class RequestTracker:
|
||||
self,
|
||||
new_block_ids: tuple[list[int], ...] | list[int],
|
||||
) -> None:
|
||||
if len(new_block_ids) == 0:
|
||||
new_block_ids = []
|
||||
elif isinstance(new_block_ids, tuple):
|
||||
new_block_ids = new_block_ids[0]
|
||||
elif isinstance(new_block_ids, list):
|
||||
pass
|
||||
else:
|
||||
raise ValueError(f"Unsupported new_block_ids type {type(new_block_ids)}")
|
||||
self.allocated_block_ids.extend(new_block_ids)
|
||||
# Backward-compat: accept a single list (broadcast to single group).
|
||||
if isinstance(new_block_ids, list):
|
||||
new_block_ids = (new_block_ids,)
|
||||
if len(new_block_ids) != len(self.allocated_block_ids):
|
||||
raise ValueError(
|
||||
f"Group count mismatch: tracker has "
|
||||
f"{len(self.allocated_block_ids)} groups, update has "
|
||||
f"{len(new_block_ids)}"
|
||||
)
|
||||
for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True):
|
||||
if new:
|
||||
existing.extend(new)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -183,7 +197,7 @@ class ReqMeta:
|
||||
|
||||
req_id: str
|
||||
token_len_chunk: int
|
||||
block_ids: list[int]
|
||||
block_ids: tuple[list[int], ...]
|
||||
block_hashes: list[BlockHash]
|
||||
|
||||
can_save: bool | None = None
|
||||
|
||||
@@ -22,7 +22,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -45,7 +47,11 @@ def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
|
||||
class MooncakeStoreScheduler:
|
||||
"""Scheduler-side component for MooncakeStoreConnector."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
@@ -56,11 +62,11 @@ class MooncakeStoreScheduler:
|
||||
self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.original_block_size = vllm_config.cache_config.block_size
|
||||
self._block_size = vllm_config.cache_config.block_size
|
||||
if self.pcp_size > 1:
|
||||
self._block_size *= self.pcp_size
|
||||
if self.dcp_size > 1:
|
||||
self._block_size *= self.dcp_size
|
||||
# LCM for multi-group HMA; bs * pcp * dcp for single-group. Matches
|
||||
# the engine's own scheduler block size by construction.
|
||||
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
|
||||
self._discard_partial_chunks = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
@@ -72,7 +78,7 @@ class MooncakeStoreScheduler:
|
||||
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
|
||||
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
|
||||
self._preempted_req_ids: set[str] = set() # preempted requests
|
||||
self._unfinished_requests: dict[str, tuple[Request, list[int]]] = {}
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
@@ -126,9 +132,9 @@ class MooncakeStoreScheduler:
|
||||
num_external_tokens: int,
|
||||
):
|
||||
"""Update state after block allocation."""
|
||||
local_block_ids: list[int] = []
|
||||
local_block_ids: tuple[list[int], ...] = ()
|
||||
if num_external_tokens > 0:
|
||||
local_block_ids = blocks.get_block_ids()[0]
|
||||
local_block_ids = blocks.get_block_ids()
|
||||
|
||||
self._unfinished_requests[request.request_id] = (request, local_block_ids)
|
||||
self._unfinished_request_ids.add(request.request_id)
|
||||
@@ -189,11 +195,12 @@ class MooncakeStoreScheduler:
|
||||
request_tuple = self._unfinished_requests.get(request.req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
if not isinstance(request.block_ids[0], list):
|
||||
unfolded_block_ids = request.block_ids.copy()
|
||||
if isinstance(request.block_ids, tuple):
|
||||
# Multi-group: preserve per-group structure.
|
||||
unfolded_block_ids = tuple(b.copy() for b in request.block_ids)
|
||||
else:
|
||||
# TODO: support HMA
|
||||
unfolded_block_ids = request.block_ids[0].copy()
|
||||
# Single-group legacy: list[int] -> 1-tuple.
|
||||
unfolded_block_ids = (request.block_ids.copy(),)
|
||||
|
||||
prefill_tokens = _new_req_prefill_tokens(request)
|
||||
request_tracker = RequestTracker(
|
||||
@@ -237,9 +244,9 @@ class MooncakeStoreScheduler:
|
||||
if req_id in self._preempted_req_ids:
|
||||
# Resumed after preemption
|
||||
if isinstance(new_block_ids, tuple):
|
||||
block_ids_list = new_block_ids[0].copy()
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
else:
|
||||
block_ids_list = new_block_ids.copy()
|
||||
new_block_ids = (new_block_ids.copy(),)
|
||||
self._preempted_req_ids.discard(req_id)
|
||||
load_spec = self.load_specs.pop(req_id, None)
|
||||
request_tuple = self._unfinished_requests.get(req_id)
|
||||
@@ -254,7 +261,7 @@ class MooncakeStoreScheduler:
|
||||
request_tracker = RequestTracker(
|
||||
req_id=req_id,
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=block_ids_list,
|
||||
allocated_block_ids=new_block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=prefill_tokens[:num_tokens_to_compute].copy(),
|
||||
prefill_end_tokens=len(prefill_tokens),
|
||||
@@ -361,7 +368,7 @@ class MooncakeStoreScheduler:
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: list[int],
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Determine whether to delay freeing blocks for async save."""
|
||||
if self.kv_role == "kv_consumer":
|
||||
@@ -370,11 +377,12 @@ class MooncakeStoreScheduler:
|
||||
assert tracker is not None
|
||||
if tracker.num_saved_tokens <= 0:
|
||||
return False, None
|
||||
delay_free_blocks = len(block_ids) > 0
|
||||
total_blocks = sum(len(g) for g in block_ids)
|
||||
delay_free_blocks = total_blocks > 0
|
||||
if delay_free_blocks:
|
||||
logger.debug(
|
||||
"Delaying free of %d blocks for request %s",
|
||||
len(block_ids),
|
||||
total_blocks,
|
||||
request.request_id,
|
||||
)
|
||||
return delay_free_blocks, None
|
||||
|
||||
@@ -10,6 +10,7 @@ Includes the store worker, transfer threads, lookup server,
|
||||
and MooncakeDistributedStore integration.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
@@ -36,15 +37,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
|
||||
get_mooncake_dp_engine_index,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
ExternalCachedBlockPool,
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
PoolKey,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import get_ip, make_zmq_socket
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
maybe_convert_block_hash,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -321,7 +332,7 @@ class KVTransferThread(threading.Thread):
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
token_database: ChunkedTokenDatabase,
|
||||
token_databases: list[ChunkedTokenDatabase],
|
||||
block_size: int,
|
||||
tp_rank: int,
|
||||
ready_event: threading.Event,
|
||||
@@ -332,7 +343,7 @@ class KVTransferThread(threading.Thread):
|
||||
self.ready_event = ready_event
|
||||
self.block_size = block_size
|
||||
self.tp_rank = tp_rank
|
||||
self.token_database = token_database
|
||||
self.token_databases = token_databases
|
||||
self.done_task_lock = threading.Lock()
|
||||
self.request_queue: queue.Queue[Any] = queue.Queue()
|
||||
self.finished_requests: set[str] = set()
|
||||
@@ -385,7 +396,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
token_database: ChunkedTokenDatabase,
|
||||
coord: MooncakeStoreCoordinator,
|
||||
token_databases: list[ChunkedTokenDatabase],
|
||||
block_size: int,
|
||||
tp_rank: int,
|
||||
put_step: int,
|
||||
@@ -396,13 +408,14 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
token_database,
|
||||
token_databases,
|
||||
block_size,
|
||||
tp_rank,
|
||||
ready_event,
|
||||
name="KVCacheStoreSendingThread",
|
||||
)
|
||||
self.put_step = put_step
|
||||
self.coord = coord
|
||||
self.kv_role = kv_role
|
||||
self.stored_requests: defaultdict[str, int] = defaultdict(int)
|
||||
self.enable_kv_event = enable_kv_event
|
||||
@@ -449,14 +462,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
return True
|
||||
|
||||
def _handle_request(self, req_meta: ReqMeta):
|
||||
token_len = req_meta.token_len_chunk
|
||||
block_ids = req_meta.block_ids
|
||||
# Cache hits are always a multiple of ``lcm_block_size`` tokens
|
||||
lcm_block_size = self.coord.lcm_block_size
|
||||
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
|
||||
block_ids_per_group = req_meta.block_ids
|
||||
req_id = req_meta.req_id
|
||||
current_event = req_meta.current_event
|
||||
|
||||
if req_id not in self.stored_requests:
|
||||
self.request_queue.task_done()
|
||||
return
|
||||
if token_len == 0:
|
||||
self.dec_stored_request(req_id)
|
||||
self.request_queue.task_done()
|
||||
return
|
||||
if self._should_skip_request(req_id):
|
||||
logger.debug(
|
||||
"Skipping Mooncake store for request %s while CPU/disk offloading "
|
||||
@@ -467,23 +486,34 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self.request_queue.task_done()
|
||||
return
|
||||
|
||||
starts = []
|
||||
ends = []
|
||||
keys = []
|
||||
# Within each lcm region only per-spec relevant chunks are loaded
|
||||
# (e.g., SWA or linear attn), so mask out irrelevant chunks
|
||||
store_masks = self.coord.store_mask(token_len)
|
||||
starts: list[int] = []
|
||||
ends: list[int] = []
|
||||
keys: list[str] = []
|
||||
block_hashes: list[BlockHash] = []
|
||||
for index, (start, end, key) in enumerate(
|
||||
self.token_database.process_tokens(token_len, req_meta.block_hashes)
|
||||
):
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
keys.append(key.to_string())
|
||||
block_hashes.append(req_meta.block_hashes[index])
|
||||
group_indices: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = store_masks[g_idx]
|
||||
for chunk_idx, (start, end, key) in enumerate(
|
||||
db.process_tokens(token_len, req_meta.block_hashes)
|
||||
):
|
||||
if chunk_idx >= len(mask) or not mask[chunk_idx]:
|
||||
continue
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
keys.append(key.to_string())
|
||||
block_hashes.append(req_meta.block_hashes[chunk_idx])
|
||||
group_indices.append(g_idx)
|
||||
|
||||
# Apply put_step striding for TP
|
||||
starts = starts[self.tp_rank % self.put_step :: self.put_step]
|
||||
ends = ends[self.tp_rank % self.put_step :: self.put_step]
|
||||
keys = keys[self.tp_rank % self.put_step :: self.put_step]
|
||||
block_hashes = block_hashes[self.tp_rank % self.put_step :: self.put_step]
|
||||
sl = slice(self.tp_rank % self.put_step, None, self.put_step)
|
||||
starts = starts[sl]
|
||||
ends = ends[sl]
|
||||
keys = keys[sl]
|
||||
block_hashes = block_hashes[sl]
|
||||
group_indices = group_indices[sl]
|
||||
|
||||
if not keys:
|
||||
self.dec_stored_request(req_id)
|
||||
@@ -501,38 +531,37 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
ends = [ends[i] for i in missing_indices]
|
||||
keys = [keys[i] for i in missing_indices]
|
||||
block_hashes = [block_hashes[i] for i in missing_indices]
|
||||
group_indices = [group_indices[i] for i in missing_indices]
|
||||
|
||||
logger.debug(
|
||||
"Storing KV cache for %d out of %d blocks "
|
||||
"(missing_count=%d) for request %s",
|
||||
"Storing KV cache for %d blocks (groups=%s) for request %s",
|
||||
len(keys),
|
||||
token_len // self.block_size,
|
||||
len(missing_indices),
|
||||
set(group_indices),
|
||||
req_id,
|
||||
)
|
||||
|
||||
addrs = []
|
||||
sizes = []
|
||||
addrs: list[list[int]] = []
|
||||
sizes: list[list[int]] = []
|
||||
stored_events: list[BlockStored] = []
|
||||
prev_key = None
|
||||
# parent_block_hash chains live within a group, not across.
|
||||
prev_key_per_group: dict[int, Any] = {}
|
||||
new_block_hashes = [maybe_convert_block_hash(bh) for bh in block_hashes]
|
||||
|
||||
for index, start in enumerate(starts):
|
||||
addr, size, _ = self.token_database.prepare_value(
|
||||
start, ends[index], block_ids
|
||||
)
|
||||
for idx, (s, e, g_idx) in enumerate(
|
||||
zip(starts, ends, group_indices, strict=True)
|
||||
):
|
||||
db = self.token_databases[g_idx]
|
||||
addr, size, _ = db.prepare_value(s, e, block_ids_per_group[g_idx])
|
||||
addrs.append(addr)
|
||||
sizes.append(size)
|
||||
|
||||
if self.enable_kv_event:
|
||||
token_ids = (
|
||||
req_meta.token_ids[start : ends[index]]
|
||||
if req_meta.token_ids is not None
|
||||
else None
|
||||
req_meta.token_ids[s:e] if req_meta.token_ids is not None else None
|
||||
)
|
||||
stored_event = BlockStored(
|
||||
block_hashes=[new_block_hashes[index]],
|
||||
parent_block_hash=prev_key,
|
||||
block_hashes=[new_block_hashes[idx]],
|
||||
parent_block_hash=prev_key_per_group.get(g_idx),
|
||||
token_ids=token_ids,
|
||||
block_size=req_meta.original_block_size,
|
||||
lora_id=None,
|
||||
@@ -540,7 +569,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
lora_name=None,
|
||||
)
|
||||
stored_events.append(stored_event)
|
||||
prev_key = new_block_hashes[index]
|
||||
prev_key_per_group[g_idx] = new_block_hashes[idx]
|
||||
|
||||
if current_event is not None:
|
||||
current_event.synchronize()
|
||||
@@ -600,7 +629,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
token_database: ChunkedTokenDatabase,
|
||||
coord: MooncakeStoreCoordinator,
|
||||
token_databases: list[ChunkedTokenDatabase],
|
||||
block_size: int,
|
||||
tp_rank: int,
|
||||
ready_event: threading.Event,
|
||||
@@ -608,7 +638,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
token_database,
|
||||
token_databases,
|
||||
block_size,
|
||||
tp_rank,
|
||||
ready_event,
|
||||
@@ -622,6 +652,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
disk_offload_buffer_budget_bytes
|
||||
)
|
||||
)
|
||||
self.coord = coord
|
||||
|
||||
def _handle_request(self, req_meta: ReqMeta):
|
||||
token_len = req_meta.load_spec.token_len # type: ignore[union-attr]
|
||||
@@ -632,32 +663,31 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
* self.block_size
|
||||
)
|
||||
|
||||
addr_list = []
|
||||
size_list = []
|
||||
key_list = []
|
||||
for start, end, key in self.token_database.process_tokens(
|
||||
token_len, req_meta.block_hashes, mask_num
|
||||
):
|
||||
addr, size, _ = self.token_database.prepare_value(
|
||||
start, end, req_meta.block_ids
|
||||
)
|
||||
key_list.append(key.to_string())
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
# Skip chunks the consumer's per-group spec wouldn't populate
|
||||
# locally (e.g. SWA pre-window) even if the producer stored them.
|
||||
load_mask_per_group = self.coord.load_mask(req_meta.block_hashes, token_len)
|
||||
|
||||
addr_list: list[list[int]] = []
|
||||
size_list: list[list[int]] = []
|
||||
key_list: list[str] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = load_mask_per_group[g_idx]
|
||||
for start, end, key in db.process_tokens(
|
||||
token_len, req_meta.block_hashes, mask_num
|
||||
):
|
||||
chunk_idx = start // db.block_size
|
||||
if chunk_idx >= len(mask) or not mask[chunk_idx]:
|
||||
continue
|
||||
addr, size, _ = db.prepare_value(start, end, req_meta.block_ids[g_idx])
|
||||
key_list.append(key.to_string())
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
|
||||
# Rotate lists by tp_rank for load balancing
|
||||
key_list_c = (
|
||||
key_list[self.tp_rank % len(key_list) :]
|
||||
+ key_list[: self.tp_rank % len(key_list)]
|
||||
)
|
||||
addr_list_c = (
|
||||
addr_list[self.tp_rank % len(addr_list) :]
|
||||
+ addr_list[: self.tp_rank % len(addr_list)]
|
||||
)
|
||||
size_list_c = (
|
||||
size_list[self.tp_rank % len(size_list) :]
|
||||
+ size_list[: self.tp_rank % len(size_list)]
|
||||
)
|
||||
rotation = self.tp_rank % len(key_list)
|
||||
key_list_c = key_list[rotation:] + key_list[:rotation]
|
||||
addr_list_c = addr_list[rotation:] + addr_list[:rotation]
|
||||
size_list_c = size_list[rotation:] + size_list[:rotation]
|
||||
|
||||
load_batches = [(key_list_c, addr_list_c, size_list_c)]
|
||||
if self.usable_disk_offload_buffer_budget_bytes is not None:
|
||||
@@ -737,7 +767,11 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
class MooncakeStoreWorker:
|
||||
"""Worker-side component for MooncakeStoreConnector."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
try:
|
||||
from mooncake.store import ( # type: ignore
|
||||
MooncakeDistributedStore,
|
||||
@@ -771,11 +805,9 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.original_block_size = self.cache_config.block_size
|
||||
self.block_size = self.cache_config.block_size
|
||||
if self.pcp_size > 1:
|
||||
self.block_size *= self.pcp_size
|
||||
if self.dcp_size > 1:
|
||||
self.block_size *= self.dcp_size
|
||||
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
self.num_layers = model_config.get_num_layers(parallel_config)
|
||||
|
||||
self.use_mla = False
|
||||
@@ -806,8 +838,6 @@ class MooncakeStoreWorker:
|
||||
pp_rank=self.pp_rank,
|
||||
)
|
||||
|
||||
self.token_database = ChunkedTokenDatabase(self.metadata, self.block_size)
|
||||
|
||||
# Initialize MooncakeDistributedStore with its own TransferEngine
|
||||
store_config = MooncakeStoreConfig.load_from_env()
|
||||
extra_config = (
|
||||
@@ -892,6 +922,45 @@ class MooncakeStoreWorker:
|
||||
self.kv_recv_thread: KVCacheStoreRecvingThread | None = None
|
||||
self.finished_store_req: set[str] = set()
|
||||
|
||||
self._kv_cache_config = kv_cache_config
|
||||
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
|
||||
# self.block_size (= scheduler_block_size) so the coordinator's
|
||||
# ``block_size % hash_block_size == 0`` invariant holds.
|
||||
groups = list(kv_cache_config.kv_cache_groups)
|
||||
if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size:
|
||||
g = groups[0]
|
||||
groups = [
|
||||
dataclasses.replace(
|
||||
g,
|
||||
kv_cache_spec=dataclasses.replace(
|
||||
g.kv_cache_spec, block_size=self.block_size
|
||||
),
|
||||
)
|
||||
]
|
||||
self._kv_cache_groups: list[KVCacheGroupSpec] = groups
|
||||
spec_cfg = getattr(vllm_config, "speculative_config", None)
|
||||
use_eagle = bool(
|
||||
spec_cfg.use_eagle()
|
||||
if spec_cfg is not None and callable(getattr(spec_cfg, "use_eagle", None))
|
||||
else False
|
||||
)
|
||||
self.coord = MooncakeStoreCoordinator(
|
||||
self._kv_cache_groups,
|
||||
scheduler_block_size=self.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
use_eagle=use_eagle,
|
||||
)
|
||||
# One ChunkedTokenDatabase per group; addresses populated in
|
||||
# register_kv_caches once the kv-cache layout is known.
|
||||
self.token_dbs: list[ChunkedTokenDatabase] = [
|
||||
ChunkedTokenDatabase(
|
||||
dataclasses.replace(self.metadata, group_id=g_idx),
|
||||
g.kv_cache_spec.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
)
|
||||
for g_idx, g in enumerate(self._kv_cache_groups)
|
||||
]
|
||||
|
||||
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
|
||||
"""Register a cross-layers KV cache tensor.
|
||||
|
||||
@@ -901,93 +970,86 @@ class MooncakeStoreWorker:
|
||||
"""
|
||||
self.register_kv_caches({"__cross_layer__": kv_cache})
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: dict[str, torch.Tensor | list[torch.Tensor]],
|
||||
) -> None:
|
||||
"""Register KV cache tensors and start transfer threads."""
|
||||
# TODO(yifan): we haven't supported HMA yet.
|
||||
first_kv_cache = next(iter(kv_caches.values()))
|
||||
if not kv_caches:
|
||||
logger.warning("No KV caches to offload.")
|
||||
return
|
||||
|
||||
# Resolve each entry to a representative tensor for storage
|
||||
# deduplication. For attention layers the value is already a tensor;
|
||||
# for Mamba layers it is a list of tensors that all share the same
|
||||
# underlying raw storage, so we take the first one.
|
||||
def _repr_tensor(v: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
|
||||
assert isinstance(v, torch.Tensor | list)
|
||||
return v if isinstance(v, torch.Tensor) else v[0]
|
||||
|
||||
# num_blocks from cache_config is authoritative (set after
|
||||
# profiling, before KV cache allocation).
|
||||
assert self.cache_config.num_gpu_blocks is not None
|
||||
self.num_blocks = self.cache_config.num_gpu_blocks
|
||||
|
||||
# Detect the KV cache memory layout using the stride-based
|
||||
# approach from simple_kv_offload/worker.py.
|
||||
#
|
||||
# The physical layout varies across attention backends:
|
||||
# FlashAttn/ROCm : (2, num_blocks, ...) → K/V outermost
|
||||
# FlashInfer/MLA : (num_blocks, ...) → blocks outermost
|
||||
#
|
||||
# We derive page_size_bytes = storage.nbytes() // num_blocks,
|
||||
# then classify dims: any dim whose byte-stride exceeds
|
||||
# page_size_bytes must be an outer segment dim (e.g. the K/V
|
||||
# dim of size 2). For those backends we register each segment
|
||||
# (K, V) as a separate base-address so that the per-block
|
||||
# offset arithmetic in prepare_value() stays correct.
|
||||
storage = first_kv_cache.untyped_storage()
|
||||
el = first_kv_cache.element_size()
|
||||
page_size_bytes = storage.nbytes() // self.num_blocks
|
||||
outer_dims = [
|
||||
d
|
||||
for d in range(first_kv_cache.ndim)
|
||||
if first_kv_cache.stride(d) * el > page_size_bytes
|
||||
]
|
||||
|
||||
# Register buffers with the store (deduplicate shared storages)
|
||||
# and record per-segment base addresses for every layer.
|
||||
seen_ptrs: set[int] = set()
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
addrs: list[int] = []
|
||||
block_lens: list[int] = []
|
||||
|
||||
for cache in kv_caches.values():
|
||||
for value in kv_caches.values():
|
||||
cache = _repr_tensor(value)
|
||||
cache_storage = cache.untyped_storage()
|
||||
base_addr = cache_storage.data_ptr()
|
||||
if base_addr in seen_ptrs:
|
||||
continue
|
||||
seen_ptrs.add(base_addr)
|
||||
region_len = cache_storage.nbytes()
|
||||
|
||||
if base_addr not in seen_ptrs:
|
||||
seen_ptrs.add(base_addr)
|
||||
ret = self.store.register_buffer(base_addr, region_len)
|
||||
if ret != 0:
|
||||
logger.error(
|
||||
"register_buffer failed for addr %#x len %d: %d",
|
||||
base_addr,
|
||||
region_len,
|
||||
ret,
|
||||
)
|
||||
ret = self.store.register_buffer(base_addr, region_len)
|
||||
if ret != 0:
|
||||
logger.error(
|
||||
"register_buffer failed for addr %#x len %d: %d",
|
||||
base_addr,
|
||||
region_len,
|
||||
ret,
|
||||
)
|
||||
|
||||
# Detect layout via stride: a dim whose byte-stride exceeds
|
||||
# page_size_bytes is an outer segment dim (e.g. the K/V dim of
|
||||
# FlashAttn's (2, num_blocks, ...)). FlashInfer/MLA's blocks-
|
||||
# outermost layout has no such dim and yields a single segment.
|
||||
el = cache.element_size()
|
||||
page_size_bytes = region_len // self.num_blocks
|
||||
outer_dims = [
|
||||
d for d in range(cache.ndim) if cache.stride(d) * el > page_size_bytes
|
||||
]
|
||||
if not outer_dims:
|
||||
# Blocks-first layout (FlashInfer / MLA): one segment.
|
||||
self.kv_caches_base_addr.append(base_addr)
|
||||
self.block_len.append(page_size_bytes)
|
||||
addrs.append(base_addr)
|
||||
block_lens.append(page_size_bytes)
|
||||
else:
|
||||
# K/V-first layout (FlashAttn / ROCm): split segments.
|
||||
seg_stride = cache.stride(outer_dims[0]) * el
|
||||
for idx in range(cache.shape[outer_dims[0]]):
|
||||
self.kv_caches_base_addr.append(base_addr + idx * seg_stride)
|
||||
self.block_len.append(seg_stride // self.num_blocks)
|
||||
addrs.append(base_addr + idx * seg_stride)
|
||||
block_lens.append(seg_stride // self.num_blocks)
|
||||
|
||||
logger.info(
|
||||
"Registering KV_Caches. use_mla: %s, shape %s, "
|
||||
"num_blocks: %d, block_len: %s, "
|
||||
"per_key_bytes: %d, "
|
||||
"num_segments: %d",
|
||||
self.use_mla,
|
||||
first_kv_cache.shape,
|
||||
"Registered KV caches: num_groups=%d, num_segments=%d, num_blocks=%d",
|
||||
len(self.token_dbs),
|
||||
len(addrs),
|
||||
self.num_blocks,
|
||||
list(set(self.block_len)),
|
||||
sum(self.block_len),
|
||||
len(self.kv_caches_base_addr),
|
||||
)
|
||||
|
||||
self.token_database.set_kv_caches_base_addr(self.kv_caches_base_addr)
|
||||
self.token_database.set_block_len(self.block_len)
|
||||
for db in self.token_dbs:
|
||||
db.set_kv_caches_base_addr(addrs)
|
||||
db.set_block_len(block_lens)
|
||||
|
||||
# Start transfer threads
|
||||
if self.kv_role in ["kv_producer", "kv_both"]:
|
||||
ready_event_sending = threading.Event()
|
||||
self.kv_send_thread = KVCacheStoreSendingThread(
|
||||
self.store,
|
||||
self.token_database,
|
||||
self.coord,
|
||||
self.token_dbs,
|
||||
self.block_size,
|
||||
self.tp_rank,
|
||||
self.put_step,
|
||||
@@ -1001,7 +1063,8 @@ class MooncakeStoreWorker:
|
||||
ready_event_recving = threading.Event()
|
||||
self.kv_recv_thread = KVCacheStoreRecvingThread(
|
||||
self.store,
|
||||
self.token_database,
|
||||
self.coord,
|
||||
self.token_dbs,
|
||||
self.block_size,
|
||||
self.tp_rank,
|
||||
ready_event_recving,
|
||||
@@ -1122,62 +1185,55 @@ class MooncakeStoreWorker:
|
||||
|
||||
return finished_sending
|
||||
|
||||
def lookup(
|
||||
self,
|
||||
token_len: int,
|
||||
block_hashes: list[BlockHash],
|
||||
) -> int:
|
||||
def lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
|
||||
"""Check how many prefix tokens exist in the store.
|
||||
|
||||
Checks across all TP ranks and PP ranks.
|
||||
"""
|
||||
end = 0
|
||||
keys: list[str] = []
|
||||
if not block_hashes or token_len <= 0:
|
||||
return 0
|
||||
|
||||
# Build per-(group, hash) candidate keys expanded across TP/PP.
|
||||
# candidate_meta[i] is the (group_id, hash_bytes) for candidate_keys[i].
|
||||
candidate_keys: list[str] = []
|
||||
candidate_meta: list[tuple[int, bytes]] = []
|
||||
tp_count = min(self.tp_size, self.num_kv_head)
|
||||
for g_idx, db in enumerate(self.token_dbs):
|
||||
spec_block_size = db.block_size
|
||||
group_hashes = self.coord.block_hashes_for_spec(
|
||||
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
|
||||
)
|
||||
for chunk_id, h in enumerate(group_hashes):
|
||||
start_idx = chunk_id * spec_block_size
|
||||
if start_idx >= token_len:
|
||||
break
|
||||
for tp in range(tp_count):
|
||||
for pp in range(self.pp_size):
|
||||
md = dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp)
|
||||
candidate_keys.append(PoolKey(md, h.hex()).to_string())
|
||||
candidate_meta.append((g_idx, bytes(h)))
|
||||
|
||||
if not candidate_keys:
|
||||
return 0
|
||||
|
||||
try:
|
||||
starts: list[int] = []
|
||||
for start, end, key in self.token_database.process_tokens(
|
||||
token_len, block_hashes
|
||||
):
|
||||
keys.append(key.to_string())
|
||||
starts.append(start)
|
||||
|
||||
# Expand keys for all TP ranks
|
||||
multi_tp_keys = keys[:]
|
||||
for i in range(1, min(self.tp_size, self.num_kv_head)):
|
||||
for item in keys:
|
||||
new_str = item.replace("@tp_rank:0", f"@tp_rank:{i}", 1)
|
||||
multi_tp_keys.append(new_str)
|
||||
|
||||
# Expand keys for all PP ranks
|
||||
pp_base_keys = multi_tp_keys.copy()
|
||||
for i in range(1, self.pp_size):
|
||||
for item in pp_base_keys:
|
||||
new_str = item.replace("@pp_rank:0", f"@pp_rank:{i}", 1)
|
||||
multi_tp_keys.append(new_str)
|
||||
|
||||
res = self.store.batch_is_exist(multi_tp_keys)
|
||||
|
||||
num_block = len(keys)
|
||||
multi_tp_values = [
|
||||
res[i * num_block : (i + 1) * num_block]
|
||||
for i in range(min(self.tp_size, self.num_kv_head) * self.pp_size)
|
||||
]
|
||||
index = self._find_min_first_non_one_index(multi_tp_values)
|
||||
if index != -1:
|
||||
return starts[index]
|
||||
res = self.store.batch_is_exist(candidate_keys)
|
||||
except Exception as e:
|
||||
logger.error("Remote connection failed in lookup: %s", e)
|
||||
return 0
|
||||
return end
|
||||
|
||||
@staticmethod
|
||||
def _find_min_first_non_one_index(
|
||||
arr: list[list[int]],
|
||||
) -> int:
|
||||
try:
|
||||
return min(idx for row in arr for idx, val in enumerate(row) if val != 1)
|
||||
except ValueError:
|
||||
return -1
|
||||
# A (group, hash) is "present" only when every TP*PP rank has it.
|
||||
expected_per_key = max(1, tp_count * self.pp_size)
|
||||
present_count: dict[tuple[int, bytes], int] = {}
|
||||
for gh, exists in zip(candidate_meta, res, strict=True):
|
||||
if exists == 1:
|
||||
present_count[gh] = present_count.get(gh, 0) + 1
|
||||
exists_set = {gh for gh, c in present_count.items() if c >= expected_per_key}
|
||||
|
||||
_masks, hit_length = self.coord.find_longest_cache_hit(
|
||||
block_hashes, token_len, ExternalCachedBlockPool(exists_set)
|
||||
)
|
||||
return hit_length
|
||||
|
||||
def get_kv_events(self) -> list[BlockStored]:
|
||||
if self.enable_kv_events and self.kv_send_thread is not None:
|
||||
@@ -1220,7 +1276,8 @@ class LookupKeyServer:
|
||||
token_len = int.from_bytes(all_frames[0], byteorder="big")
|
||||
hash_frames = all_frames[1:]
|
||||
hashes_str = self.decoder.decode(hash_frames)
|
||||
result = self.store_worker.lookup(token_len, hashes_str)
|
||||
block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str]
|
||||
result = self.store_worker.lookup(token_len, block_hashes)
|
||||
response = result.to_bytes(4, "big")
|
||||
self.socket.send(response)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user