[KV Connector] Support heterogeneous TP sharing in Mooncake Store Connector (#53129)
Signed-off-by: Guanyi Chen <939416532@qq.com> Signed-off-by: z-zanez <zhouzh93@mail2.sysu.edu.cn> Signed-off-by: Zheng Zhou <zhouzh93@mail2.sysu.edu.cn> Signed-off-by: zane <zhouzh93@mail2.sysu.edu.cn> Co-authored-by: Guanyi Chen <939416532@qq.com>
This commit is contained in:
@@ -11,16 +11,17 @@ and consumer instances read/write KV to/from the store independently,
|
||||
enabling prefix caching via hash-based deduplication.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_events import (
|
||||
BlockStored,
|
||||
KVCacheEvent,
|
||||
KVConnectorKVEvents,
|
||||
KVEventAggregator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
@@ -56,31 +57,66 @@ logger = init_logger(__name__)
|
||||
class MooncakeStoreKVEvents(KVConnectorKVEvents):
|
||||
"""KV event aggregation for MooncakeStoreConnector."""
|
||||
|
||||
def __init__(self, num_workers: int) -> None:
|
||||
self._aggregator = KVEventAggregator(num_workers)
|
||||
def __init__(
|
||||
self,
|
||||
num_workers: int,
|
||||
group_tp_replication_factors: Sequence[int] = (1,),
|
||||
) -> None:
|
||||
if num_workers <= 0:
|
||||
raise ValueError("num_workers must be greater than zero.")
|
||||
if any(factor <= 0 for factor in group_tp_replication_factors):
|
||||
raise ValueError("TP replication factors must be greater than zero.")
|
||||
self._event_counter: Counter[KVCacheEvent] = Counter()
|
||||
self._num_workers = num_workers
|
||||
self._group_tp_replication_factors = tuple(group_tp_replication_factors)
|
||||
|
||||
def add_events(self, events: list[KVCacheEvent]) -> None:
|
||||
self._aggregator.add_events(events)
|
||||
if not isinstance(events, list):
|
||||
raise TypeError("events must be a list of KVCacheEvent.")
|
||||
self._event_counter.update(events)
|
||||
|
||||
def _replication_factor(self, event: KVCacheEvent) -> int:
|
||||
if not isinstance(event, BlockStored) or event.group_idx is None:
|
||||
return 1
|
||||
return self._group_tp_replication_factors[event.group_idx]
|
||||
|
||||
def _is_common_event(self, event: KVCacheEvent, count: int) -> bool:
|
||||
return count * self._replication_factor(event) >= self._num_workers
|
||||
|
||||
def aggregate(self) -> "MooncakeStoreKVEvents":
|
||||
common_events = self._aggregator.get_common_events()
|
||||
self._aggregator.clear_events()
|
||||
self._aggregator.add_events(common_events)
|
||||
self._aggregator.reset_workers()
|
||||
common_events = self.pop_common_events()
|
||||
self._event_counter.clear()
|
||||
self._event_counter.update(common_events)
|
||||
self._num_workers = 1
|
||||
return self
|
||||
|
||||
def pop_common_events(self) -> list[KVCacheEvent]:
|
||||
common_events = [
|
||||
event
|
||||
for event, count in self._event_counter.items()
|
||||
if self._is_common_event(event, count)
|
||||
]
|
||||
for event in common_events:
|
||||
del self._event_counter[event]
|
||||
return common_events
|
||||
|
||||
def has_events(self) -> bool:
|
||||
return bool(self._event_counter)
|
||||
|
||||
def increment_workers(self, count: int = 1) -> None:
|
||||
self._aggregator.increment_workers(count)
|
||||
if count <= 0:
|
||||
raise ValueError("count must be positive.")
|
||||
self._num_workers += count
|
||||
|
||||
def get_all_events(self) -> list[KVCacheEvent]:
|
||||
return self._aggregator.get_all_events()
|
||||
return list(self._event_counter.elements())
|
||||
|
||||
def get_number_of_workers(self) -> int:
|
||||
return self._aggregator.get_number_of_workers()
|
||||
return self._num_workers
|
||||
|
||||
def clear_events(self) -> None:
|
||||
self._aggregator.clear_events()
|
||||
self._aggregator.reset_workers()
|
||||
self._event_counter.clear()
|
||||
self._num_workers = 1
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
|
||||
@@ -260,16 +296,13 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self._kv_cache_events = kv_cache_events
|
||||
else:
|
||||
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
|
||||
self._kv_cache_events.increment_workers(
|
||||
kv_cache_events.get_number_of_workers()
|
||||
)
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self._kv_cache_events is not None:
|
||||
self._kv_cache_events.aggregate()
|
||||
yield from self._kv_cache_events.get_all_events()
|
||||
self._kv_cache_events.clear_events()
|
||||
self._kv_cache_events = None
|
||||
events = self._kv_cache_events.pop_common_events()
|
||||
if not self._kv_cache_events.has_events():
|
||||
self._kv_cache_events = None
|
||||
yield from events
|
||||
|
||||
# ============================================================
|
||||
# Worker-side methods
|
||||
@@ -321,11 +354,19 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self,
|
||||
) -> MooncakeStoreKVEvents | None:
|
||||
assert self.connector_worker is not None
|
||||
events = self.connector_worker.get_kv_events()
|
||||
if not events:
|
||||
if (
|
||||
not self.connector_worker.enable_kv_events
|
||||
or self.connector_worker.kv_send_thread is None
|
||||
):
|
||||
return None
|
||||
|
||||
kv_events = MooncakeStoreKVEvents(num_workers=1)
|
||||
events = self.connector_worker.get_kv_events()
|
||||
# Empty containers still count this worker toward the poll's quorum.
|
||||
kv_events = MooncakeStoreKVEvents(
|
||||
num_workers=1,
|
||||
group_tp_replication_factors=(
|
||||
self.connector_worker.group_tp_replication_factors
|
||||
),
|
||||
)
|
||||
kv_events.add_events(events)
|
||||
return kv_events
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import is_non_overlapping_and_dense
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashListWithBlockSize,
|
||||
@@ -111,6 +112,9 @@ class KeyMetadata:
|
||||
# share one Mooncake master without colliding on identical block hashes.
|
||||
# Empty (the default) keeps keys byte-identical to the unprefixed format.
|
||||
cache_prefix: str = ""
|
||||
# Complete namespace for an opt-in Store payload format. Empty keeps
|
||||
# historical keys byte-identical.
|
||||
store_namespace: str = ""
|
||||
|
||||
|
||||
@dataclass(order=True)
|
||||
@@ -125,6 +129,7 @@ class PoolKey:
|
||||
(
|
||||
self.key_metadata.cache_prefix,
|
||||
self.key_metadata.model_name,
|
||||
self.key_metadata.store_namespace,
|
||||
self.key_metadata.tp_rank,
|
||||
self.key_metadata.pcp_rank,
|
||||
self.key_metadata.dcp_rank,
|
||||
@@ -148,6 +153,7 @@ class PoolKey:
|
||||
return (
|
||||
f"{prefix}"
|
||||
f"{key_metadata.model_name}"
|
||||
f"{key_metadata.store_namespace}"
|
||||
f"@tp_rank:{key_metadata.tp_rank if tp_rank is None else tp_rank}"
|
||||
f"@pcp{key_metadata.pcp_rank if pcp_rank is None else pcp_rank}"
|
||||
f"@dcp{key_metadata.dcp_rank if dcp_rank is None else dcp_rank}"
|
||||
@@ -165,61 +171,149 @@ class PoolKey:
|
||||
)
|
||||
|
||||
|
||||
class ChunkedTokenDatabase:
|
||||
"""Maps token positions to store keys and GPU memory addresses."""
|
||||
StoreShardId = int
|
||||
# Physical ranks encoded in a rank-local key: (TP, PCP, DCP, PP).
|
||||
RankNamespace = tuple[int, int, int, int]
|
||||
|
||||
|
||||
class StoreLayout:
|
||||
"""Store payload layout for one KV-cache group on one local TP rank."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int | None = None,
|
||||
):
|
||||
hash_block_size: int,
|
||||
) -> None:
|
||||
self.metadata = metadata
|
||||
self.block_size = block_size
|
||||
self.hash_block_size = hash_block_size or block_size
|
||||
if self.block_size % self.hash_block_size != 0:
|
||||
raise ValueError(
|
||||
f"block_size ({self.block_size}) must be a multiple of "
|
||||
f"hash_block_size ({self.hash_block_size})"
|
||||
)
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
self._key_prefix = PoolKey.build_prefix(metadata)
|
||||
self.hash_block_size = hash_block_size
|
||||
|
||||
def key_for(self, chunk_hash: BlockHash) -> str:
|
||||
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
|
||||
@property
|
||||
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
|
||||
"""Store objects contributed or consumed by this local rank."""
|
||||
raise NotImplementedError
|
||||
|
||||
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
|
||||
self.kv_caches_base_addr = kv_caches_base_addr
|
||||
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def set_block_len(self, block_len: list[int]):
|
||||
self.block_len = block_len
|
||||
def lookup_key_prefixes(
|
||||
self,
|
||||
rank_namespaces: Sequence[RankNamespace],
|
||||
) -> tuple[str, ...]:
|
||||
"""Prefixes that must exist for a logical block to be reusable."""
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare_value(
|
||||
self, start: int, end: int, block_ids: list[int]
|
||||
) -> tuple[list[int], list[int], int]:
|
||||
"""Compute memory addresses and sizes for a single token range.
|
||||
|
||||
Returns:
|
||||
(addr_list, size_list, block_id)
|
||||
"""
|
||||
addr_lists, size_lists, chunk_block_ids = self.prepare_values(
|
||||
((start, end),), block_ids
|
||||
)
|
||||
return addr_lists[0], size_lists[0], chunk_block_ids[0]
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: Sequence[torch.Tensor],
|
||||
num_blocks: int,
|
||||
) -> None:
|
||||
"""Build descriptors from the local KV cache tensors."""
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
shard_ids: Sequence[StoreShardId],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
"""Compute memory addresses and sizes for multiple token ranges.
|
||||
"""Build Store multi-buffer descriptors for logical chunks."""
|
||||
raise NotImplementedError
|
||||
|
||||
Returns:
|
||||
(addr_lists, size_lists, chunk_block_ids), one entry per chunk.
|
||||
"""
|
||||
|
||||
class RankLocalStoreLayout(StoreLayout):
|
||||
"""Historical rank-local Store payload layout."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int,
|
||||
) -> None:
|
||||
super().__init__(metadata, block_size, hash_block_size)
|
||||
self._key_prefix = PoolKey.build_prefix(metadata)
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
|
||||
@property
|
||||
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
|
||||
return (0,)
|
||||
|
||||
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
|
||||
assert shard_id == 0
|
||||
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
|
||||
|
||||
def lookup_key_prefixes(
|
||||
self,
|
||||
rank_namespaces: Sequence[RankNamespace],
|
||||
) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
PoolKey.build_prefix(
|
||||
self.metadata,
|
||||
tp_rank=tp_rank,
|
||||
pcp_rank=pcp_rank,
|
||||
dcp_rank=dcp_rank,
|
||||
pp_rank=pp_rank,
|
||||
)
|
||||
for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces
|
||||
)
|
||||
|
||||
def set_kv_caches_base_addr(self, base_addrs: list[int]) -> None:
|
||||
self.kv_caches_base_addr = base_addrs
|
||||
|
||||
def set_block_len(self, block_lens: list[int]) -> None:
|
||||
self.block_len = block_lens
|
||||
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: Sequence[torch.Tensor],
|
||||
num_blocks: int,
|
||||
) -> None:
|
||||
seen_region_ptrs: set[int] = set()
|
||||
base_addrs: list[int] = []
|
||||
block_lens: list[int] = []
|
||||
for cache in kv_caches:
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
storage_len = storage.nbytes()
|
||||
if not is_non_overlapping_and_dense(cache[0]):
|
||||
for head_idx in range(cache.shape[1]):
|
||||
head_cache = cache[:, head_idx]
|
||||
assert is_non_overlapping_and_dense(head_cache[0])
|
||||
region_addr = head_cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
base_addrs.append(region_addr)
|
||||
block_lens.append(head_cache.stride(0) * head_cache.element_size())
|
||||
elif cache.stride(0) * cache.element_size() * num_blocks == storage_len:
|
||||
if storage_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(storage_addr)
|
||||
base_addrs.append(storage_addr)
|
||||
block_lens.append(storage_len // num_blocks)
|
||||
else:
|
||||
region_addr = cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
base_addrs.append(region_addr)
|
||||
block_lens.append(cache.stride(0) * cache.element_size())
|
||||
self.kv_caches_base_addr = base_addrs
|
||||
self.block_len = block_lens
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
shard_ids: Sequence[StoreShardId],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
if not chunks:
|
||||
return [], [], []
|
||||
assert len(chunks) == len(shard_ids) and all(
|
||||
shard_id == 0 for shard_id in shard_ids
|
||||
)
|
||||
base = np.asarray(self.kv_caches_base_addr, dtype=np.int64)
|
||||
length = len(self.block_len)
|
||||
blen = np.asarray(
|
||||
@@ -229,7 +323,7 @@ class ChunkedTokenDatabase:
|
||||
n = len(chunks)
|
||||
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
|
||||
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
|
||||
assert not (spans % self.hash_block_size).any()
|
||||
assert np.all(spans % self.hash_block_size == 0)
|
||||
bids = np.fromiter(
|
||||
(block_ids[i] for i in (starts // self.block_size).tolist()),
|
||||
dtype=np.int64,
|
||||
@@ -241,15 +335,272 @@ class ChunkedTokenDatabase:
|
||||
return addrs.tolist(), sizes.tolist(), bids.tolist()
|
||||
|
||||
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
|
||||
"""Return addresses and sizes for one physical block slot."""
|
||||
addr_list = []
|
||||
size_list = []
|
||||
length = len(self.block_len)
|
||||
for index, base_addr in enumerate(self.kv_caches_base_addr):
|
||||
addr = base_addr + block_id * self.block_len[index % length]
|
||||
addr_list.append(addr)
|
||||
size_list.append(self.block_len[index % length])
|
||||
return addr_list, size_list
|
||||
return (
|
||||
[
|
||||
base_addr + block_id * self.block_len[index % length]
|
||||
for index, base_addr in enumerate(self.kv_caches_base_addr)
|
||||
],
|
||||
[
|
||||
self.block_len[index % length]
|
||||
for index in range(len(self.kv_caches_base_addr))
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TPShardedStoreLayout(StoreLayout):
|
||||
"""Store layout shared by divisible TP sizes."""
|
||||
|
||||
store_format: str
|
||||
|
||||
@classmethod
|
||||
def shared_namespace(cls, store_tp_size: int, pp_size: int) -> str:
|
||||
return (
|
||||
f"@store_tp:{store_tp_size}@store_pp:{pp_size}"
|
||||
f"@store_format:{cls.store_format}"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int,
|
||||
local_tp_size: int,
|
||||
store_tp_size: int,
|
||||
tp_rank: int,
|
||||
num_kv_heads: int,
|
||||
) -> None:
|
||||
super().__init__(metadata, block_size, hash_block_size)
|
||||
self.shards_per_rank = store_tp_size // local_tp_size
|
||||
first_shard = tp_rank * self.shards_per_rank
|
||||
self.store_shard_ids = tuple(
|
||||
range(first_shard, first_shard + self.shards_per_rank)
|
||||
)
|
||||
self.store_tp_size = store_tp_size
|
||||
self.heads_per_store_shard = num_kv_heads // store_tp_size
|
||||
self.local_num_kv_heads = num_kv_heads // local_tp_size
|
||||
self._shard_key_prefixes = {
|
||||
shard_id: PoolKey.build_prefix(metadata, tp_rank=shard_id)
|
||||
for shard_id in self.store_shard_ids
|
||||
}
|
||||
self._shard_addr_bases = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
|
||||
self._shard_block_strides = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
|
||||
self._shard_sizes = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
|
||||
|
||||
@property
|
||||
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
|
||||
return self.store_shard_ids
|
||||
|
||||
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
|
||||
return PoolKey.build_key_string(
|
||||
self._shard_key_prefixes[shard_id], chunk_hash.hex()
|
||||
)
|
||||
|
||||
def lookup_key_prefixes(
|
||||
self,
|
||||
rank_namespaces: Sequence[RankNamespace],
|
||||
) -> tuple[str, ...]:
|
||||
pp_ranks = sorted({namespace[3] for namespace in rank_namespaces})
|
||||
return tuple(
|
||||
PoolKey.build_prefix(self.metadata, tp_rank=shard_id, pp_rank=pp_rank)
|
||||
for shard_id in range(self.store_tp_size)
|
||||
for pp_rank in pp_ranks
|
||||
)
|
||||
|
||||
def _set_segment_templates(
|
||||
self, templates: Sequence[tuple[list[int], list[int], list[int]]]
|
||||
) -> None:
|
||||
self._shard_addr_bases = np.asarray(
|
||||
[template[0] for template in templates], dtype=np.uint64
|
||||
)
|
||||
self._shard_block_strides = np.asarray(
|
||||
[template[1] for template in templates], dtype=np.uint64
|
||||
)
|
||||
self._shard_sizes = np.asarray(
|
||||
[template[2] for template in templates], dtype=np.uint64
|
||||
)
|
||||
|
||||
def _has_packed_strides(
|
||||
self, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
def _shard_segments(
|
||||
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> Iterable[tuple[int, int]]:
|
||||
raise NotImplementedError
|
||||
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: Sequence[torch.Tensor],
|
||||
num_blocks: int,
|
||||
) -> None:
|
||||
templates: list[tuple[list[int], list[int], list[int]]] = [
|
||||
([], [], []) for _ in range(self.shards_per_rank)
|
||||
]
|
||||
for cache in kv_caches:
|
||||
if cache.ndim != 4 or tuple(cache.shape[:3]) != (
|
||||
num_blocks,
|
||||
self.local_num_kv_heads,
|
||||
self.block_size,
|
||||
):
|
||||
raise ValueError(
|
||||
"TP-shared Mooncake store requires packed KV caches with "
|
||||
"logical shape (num_blocks, local_kv_heads, block_size, content)"
|
||||
)
|
||||
|
||||
element_size = cache.element_size()
|
||||
block_stride, head_stride, token_stride, content_stride = (
|
||||
stride * element_size for stride in cache.stride()
|
||||
)
|
||||
content_bytes = cache.shape[3] * element_size
|
||||
if content_stride != element_size or not self._has_packed_strides(
|
||||
head_stride, token_stride, content_bytes
|
||||
):
|
||||
raise ValueError(
|
||||
"TP-shared Mooncake store requires packed "
|
||||
f"{self.store_format} KV layout"
|
||||
)
|
||||
|
||||
for local_shard, (addr_bases, block_strides, sizes) in enumerate(templates):
|
||||
head_start = local_shard * self.heads_per_store_shard
|
||||
for offset, size in self._shard_segments(
|
||||
head_start, head_stride, token_stride, content_bytes
|
||||
):
|
||||
addr_bases.append(cache.data_ptr() + offset)
|
||||
block_strides.append(block_stride)
|
||||
sizes.append(size)
|
||||
|
||||
self._set_segment_templates(templates)
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
shard_ids: Sequence[StoreShardId],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
if not chunks:
|
||||
return [], [], []
|
||||
if len(chunks) != len(shard_ids):
|
||||
raise ValueError("Each Store chunk must have one shard ID")
|
||||
|
||||
count = len(chunks)
|
||||
starts = np.fromiter(
|
||||
(chunk[0] for chunk in chunks), dtype=np.int64, count=count
|
||||
)
|
||||
ends = np.fromiter((chunk[1] for chunk in chunks), dtype=np.int64, count=count)
|
||||
if np.any(ends - starts != self.block_size):
|
||||
raise ValueError("TP-shared Mooncake store requires full KV blocks")
|
||||
|
||||
first_shard = self.store_shard_ids[0]
|
||||
local_shards = np.fromiter(
|
||||
(shard_id - first_shard for shard_id in shard_ids),
|
||||
dtype=np.int64,
|
||||
count=count,
|
||||
)
|
||||
if np.any(local_shards < 0) or np.any(local_shards >= self.shards_per_rank):
|
||||
raise ValueError("Store shard is not owned by this TP rank")
|
||||
chunk_block_ids = np.fromiter(
|
||||
(block_ids[index] for index in (starts // self.block_size).tolist()),
|
||||
dtype=np.uint64,
|
||||
count=count,
|
||||
)
|
||||
addr_bases = self._shard_addr_bases[local_shards]
|
||||
block_strides = self._shard_block_strides[local_shards]
|
||||
addrs = addr_bases + chunk_block_ids[:, None] * block_strides
|
||||
sizes = self._shard_sizes[local_shards]
|
||||
return addrs.tolist(), sizes.tolist(), chunk_block_ids.tolist()
|
||||
|
||||
|
||||
class LBHNCStoreLayout(TPShardedStoreLayout):
|
||||
"""Native head-major layout shared by divisible TP sizes."""
|
||||
|
||||
store_format = "tp_shared_lbhnc"
|
||||
|
||||
def _has_packed_strides(
|
||||
self, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> bool:
|
||||
return (
|
||||
token_stride == content_bytes
|
||||
and head_stride == self.block_size * content_bytes
|
||||
)
|
||||
|
||||
def _shard_segments(
|
||||
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> Iterable[tuple[int, int]]:
|
||||
yield head_start * head_stride, self.heads_per_store_shard * head_stride
|
||||
|
||||
|
||||
class LBNHCStoreLayout(TPShardedStoreLayout):
|
||||
"""Native token-major layout shared by divisible TP sizes."""
|
||||
|
||||
store_format = "tp_shared_lbnhc"
|
||||
|
||||
def _has_packed_strides(
|
||||
self, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> bool:
|
||||
return (
|
||||
head_stride == content_bytes
|
||||
and token_stride == self.local_num_kv_heads * content_bytes
|
||||
)
|
||||
|
||||
def _shard_segments(
|
||||
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> Iterable[tuple[int, int]]:
|
||||
for token_idx in range(self.block_size):
|
||||
yield (
|
||||
token_idx * token_stride + head_start * head_stride,
|
||||
self.heads_per_store_shard * content_bytes,
|
||||
)
|
||||
|
||||
|
||||
class ChunkedTokenDatabase:
|
||||
"""Enumerates logical token chunks and their hashes."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int | None = None,
|
||||
store_layout: StoreLayout | None = None,
|
||||
):
|
||||
self.metadata = metadata
|
||||
self.block_size = block_size
|
||||
self.hash_block_size = hash_block_size or block_size
|
||||
if self.block_size % self.hash_block_size != 0:
|
||||
raise ValueError(
|
||||
f"block_size ({self.block_size}) must be a multiple of "
|
||||
f"hash_block_size ({self.hash_block_size})"
|
||||
)
|
||||
self.store_layout = store_layout or RankLocalStoreLayout(
|
||||
metadata, block_size, self.hash_block_size
|
||||
)
|
||||
|
||||
@property
|
||||
def kv_caches_base_addr(self) -> list[int]:
|
||||
return self._rank_local_layout().kv_caches_base_addr
|
||||
|
||||
@property
|
||||
def block_len(self) -> list[int]:
|
||||
return self._rank_local_layout().block_len
|
||||
|
||||
def _rank_local_layout(self) -> RankLocalStoreLayout:
|
||||
if not isinstance(self.store_layout, RankLocalStoreLayout):
|
||||
raise RuntimeError("This operation requires a rank-local Store layout")
|
||||
return self.store_layout
|
||||
|
||||
def key_for(self, chunk_hash: BlockHash) -> str:
|
||||
return self._rank_local_layout().key_for(0, chunk_hash)
|
||||
|
||||
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
|
||||
self._rank_local_layout().set_kv_caches_base_addr(kv_caches_base_addr)
|
||||
|
||||
def set_block_len(self, block_len: list[int]):
|
||||
self._rank_local_layout().set_block_len(block_len)
|
||||
|
||||
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
|
||||
"""Return addresses and sizes for one physical block slot."""
|
||||
return self._rank_local_layout().prepare_value_for_block(block_id)
|
||||
|
||||
def process_tokens(
|
||||
self,
|
||||
|
||||
@@ -12,6 +12,7 @@ and MooncakeDistributedStore integration.
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
import socket
|
||||
@@ -44,12 +45,16 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
BlobBlockHashes,
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LBHNCStoreLayout,
|
||||
LBNHCStoreLayout,
|
||||
MooncakeLookupResult,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
PoolKey,
|
||||
ReqMeta,
|
||||
StoreShardId,
|
||||
TailKeyBoundary,
|
||||
TPShardedStoreLayout,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import ( # noqa: E501
|
||||
LOOKUP_MSG,
|
||||
@@ -62,7 +67,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.network_utils import get_ip, make_zmq_socket
|
||||
from vllm.utils.torch_utils import is_non_overlapping_and_dense
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
@@ -71,6 +75,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
@@ -79,6 +84,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
UniformTypeKVCacheSpecs,
|
||||
group_kernel_blocks,
|
||||
)
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
from .metrics import MooncakeStoreConnectorStats
|
||||
|
||||
@@ -92,6 +98,22 @@ MOONCAKE_NO_AVAILABLE_HANDLE = -200
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
def resolve_store_tp_size(extra_config: dict[str, Any]) -> int | None:
|
||||
"""Resolve the common Store TP requested by connector config."""
|
||||
if extra_config.get("enable_store_tp_lcm") is True:
|
||||
prefill_tp_sizes = extra_config.get("prefill_tp_sizes")
|
||||
if not isinstance(prefill_tp_sizes, list) or not prefill_tp_sizes:
|
||||
return None
|
||||
if any(
|
||||
type(tp_size) is not int or tp_size <= 0 for tp_size in prefill_tp_sizes
|
||||
):
|
||||
return None
|
||||
return math.lcm(*prefill_tp_sizes)
|
||||
|
||||
store_tp_size = extra_config.get("store_tp_size")
|
||||
return store_tp_size if type(store_tp_size) is int and store_tp_size > 0 else None
|
||||
|
||||
|
||||
def _rotate_list(values: list[_T], offset: int) -> list[_T]:
|
||||
return values[offset:] + values[:offset]
|
||||
|
||||
@@ -109,7 +131,10 @@ def _make_mooncake_group_id(metadata: KeyMetadata, chunk_hash: str) -> str:
|
||||
# Mooncake group ids describe the lifecycle unit. For vLLM, that unit is
|
||||
# a prefix chunk, so shard dimensions stay only in the object key.
|
||||
prefix = f"{metadata.cache_prefix}@" if metadata.cache_prefix else ""
|
||||
return f"vllm-mooncake-store:{prefix}{metadata.model_name}@{chunk_hash}"
|
||||
return (
|
||||
f"vllm-mooncake-store:{prefix}{metadata.model_name}"
|
||||
f"{metadata.store_namespace}@{chunk_hash}"
|
||||
)
|
||||
|
||||
|
||||
# Mirrors FileStorageConfig::local_buffer_size in Mooncake C++.
|
||||
@@ -952,8 +977,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
starts: list[int] = []
|
||||
ends: list[int] = []
|
||||
keys: list[str] = []
|
||||
kv_event_block_hashes: list[BlockHash] = []
|
||||
event_specs: list[tuple[int, int, int, BlockHash]] | None = (
|
||||
[] if self.enable_kv_event else None
|
||||
)
|
||||
group_indices: list[int] = []
|
||||
store_shard_ids: list[StoreShardId] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
# Rotate the stride phase per group to balance load across ranks.
|
||||
put_step = self.group_put_steps[g_idx]
|
||||
@@ -980,12 +1008,14 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
block_idx,
|
||||
)
|
||||
continue
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
keys.append(db.key_for(block_hash))
|
||||
if self.enable_kv_event:
|
||||
kv_event_block_hashes.append(block_hash)
|
||||
group_indices.append(g_idx)
|
||||
for store_shard_id in db.store_layout.local_shard_ids:
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
keys.append(db.store_layout.key_for(store_shard_id, block_hash))
|
||||
group_indices.append(g_idx)
|
||||
store_shard_ids.append(store_shard_id)
|
||||
if event_specs is not None:
|
||||
event_specs.append((start, end, g_idx, block_hash))
|
||||
|
||||
if not keys:
|
||||
self._record_saved(req_meta, token_len)
|
||||
@@ -1023,11 +1053,10 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
starts = [starts[i] for i in missing_indices]
|
||||
ends = [ends[i] for i in missing_indices]
|
||||
keys = [keys[i] for i in missing_indices]
|
||||
if self.enable_kv_event:
|
||||
kv_event_block_hashes = [
|
||||
kv_event_block_hashes[i] for i in missing_indices
|
||||
]
|
||||
if event_specs is not None:
|
||||
event_specs = [event_specs[i] for i in missing_indices]
|
||||
group_indices = [group_indices[i] for i in missing_indices]
|
||||
store_shard_ids = [store_shard_ids[i] for i in missing_indices]
|
||||
|
||||
group_ids = (
|
||||
[
|
||||
@@ -1050,61 +1079,33 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
|
||||
addrs: list[list[int]] = []
|
||||
sizes: list[list[int]] = []
|
||||
stored_events: list[BlockStored] = []
|
||||
chunks_per_group: list[list[tuple[int, int]]] = [
|
||||
[] for _ in self.token_databases
|
||||
]
|
||||
for start, end, g_idx in zip(starts, ends, group_indices, strict=True):
|
||||
shards_per_group: list[list[StoreShardId]] = [
|
||||
[] for _ in self.token_databases
|
||||
]
|
||||
for start, end, g_idx, store_shard_id in zip(
|
||||
starts,
|
||||
ends,
|
||||
group_indices,
|
||||
store_shard_ids,
|
||||
strict=True,
|
||||
):
|
||||
chunks_per_group[g_idx].append((start, end))
|
||||
shards_per_group[g_idx].append(store_shard_id)
|
||||
for g_idx, chunks in enumerate(chunks_per_group):
|
||||
if not chunks:
|
||||
continue
|
||||
db = self.token_databases[g_idx]
|
||||
group_addrs, group_sizes, _ = db.prepare_values(
|
||||
chunks, block_ids_per_group[g_idx]
|
||||
group_addrs, group_sizes, _ = db.store_layout.prepare_values(
|
||||
chunks,
|
||||
block_ids_per_group[g_idx],
|
||||
shards_per_group[g_idx],
|
||||
)
|
||||
addrs.extend(group_addrs)
|
||||
sizes.extend(group_sizes)
|
||||
|
||||
if self.enable_kv_event:
|
||||
new_block_hashes = [
|
||||
maybe_convert_block_hash(bh) for bh in kv_event_block_hashes
|
||||
]
|
||||
token_ids_end = token_ids_start + len(event_token_ids or ())
|
||||
|
||||
for idx, (s, e, g_idx) in enumerate(
|
||||
zip(starts, ends, group_indices, strict=True)
|
||||
):
|
||||
db = self.token_databases[g_idx]
|
||||
if self.enable_kv_event:
|
||||
token_ids = (
|
||||
event_token_ids[s - token_ids_start : e - token_ids_start]
|
||||
if event_token_ids is not None
|
||||
and token_ids_start <= s
|
||||
and e <= token_ids_end
|
||||
else []
|
||||
)
|
||||
stored_event = BlockStored(
|
||||
block_hashes=[new_block_hashes[idx]],
|
||||
# Derive the direct predecessor from the unfiltered
|
||||
# request chain. Adjacent PUTs need not be adjacent in
|
||||
# that chain after Store dedup, masks, or TP striding.
|
||||
parent_block_hash=(
|
||||
maybe_convert_block_hash(
|
||||
req_meta.block_hashes[s // db.hash_block_size - 1]
|
||||
)
|
||||
if s > 0
|
||||
else None
|
||||
),
|
||||
token_ids=token_ids,
|
||||
block_size=db.block_size,
|
||||
lora_id=None,
|
||||
medium="cpu",
|
||||
lora_name=None,
|
||||
group_idx=g_idx,
|
||||
)
|
||||
stored_events.append(stored_event)
|
||||
|
||||
if current_event is not None:
|
||||
current_event.synchronize()
|
||||
|
||||
@@ -1112,6 +1113,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
assert len(group_ids) == len(keys)
|
||||
self.replicate_config.group_ids = group_ids
|
||||
|
||||
failed_indices: set[int] = set()
|
||||
put_had_exception = False
|
||||
batch_bytes = _sum_batch_bytes(sizes)
|
||||
put_start = time.perf_counter()
|
||||
try:
|
||||
@@ -1121,54 +1124,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
sizes,
|
||||
self.replicate_config,
|
||||
)
|
||||
failed = [i for i, v in enumerate(res) if v < 0]
|
||||
self._record_operation(
|
||||
"save_put",
|
||||
put_start,
|
||||
len(keys),
|
||||
num_bytes=batch_bytes,
|
||||
status="partial_failure" if failed else "ok",
|
||||
num_failed_keys=len(failed),
|
||||
)
|
||||
if failed:
|
||||
failed_codes = set(res[i] for i in failed)
|
||||
if self.enable_kv_event:
|
||||
failed_indices = set(failed)
|
||||
stored_events = [
|
||||
event
|
||||
for i, event in enumerate(stored_events)
|
||||
if i not in failed_indices
|
||||
]
|
||||
logger.warning(
|
||||
"batch_put failed: %d/%d keys failed "
|
||||
"(codes=%s, batch_bytes=%d, num_keys=%d), "
|
||||
"first_key=%s",
|
||||
len(failed),
|
||||
len(keys),
|
||||
failed_codes,
|
||||
batch_bytes,
|
||||
len(keys),
|
||||
keys[0] if keys else "N/A",
|
||||
)
|
||||
if (
|
||||
MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes
|
||||
and not self._mark_request_skipped_for_pressure(req_meta)
|
||||
):
|
||||
logger.warning(
|
||||
"Detected Mooncake CPU/disk offloading pressure "
|
||||
"(NO_AVAILABLE_HANDLE); skipping future store "
|
||||
"batches for request %s until a later store "
|
||||
"batch succeeds",
|
||||
req_id,
|
||||
)
|
||||
else:
|
||||
self._record_saved(req_meta, token_len)
|
||||
save_completed = True
|
||||
if self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared "
|
||||
"after a successful store batch"
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_operation(
|
||||
"save_put",
|
||||
@@ -1179,7 +1134,96 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
num_failed_keys=len(keys),
|
||||
)
|
||||
logger.error("Failed to put key %s, error: %s", keys, e)
|
||||
stored_events.clear()
|
||||
put_had_exception = True
|
||||
else:
|
||||
failed_indices = {i for i, value in enumerate(res) if value < 0}
|
||||
self._record_operation(
|
||||
"save_put",
|
||||
put_start,
|
||||
len(keys),
|
||||
num_bytes=batch_bytes,
|
||||
status="partial_failure" if failed_indices else "ok",
|
||||
num_failed_keys=len(failed_indices),
|
||||
)
|
||||
failed_codes = {res[i] for i in failed_indices}
|
||||
if failed_indices:
|
||||
logger.warning(
|
||||
"batch_put failed: %d/%d keys failed "
|
||||
"(codes=%s, batch_bytes=%d), first_key=%s",
|
||||
len(failed_indices),
|
||||
len(keys),
|
||||
failed_codes,
|
||||
batch_bytes,
|
||||
keys[0],
|
||||
)
|
||||
if (
|
||||
MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes
|
||||
and not self._mark_request_skipped_for_pressure(req_meta)
|
||||
):
|
||||
logger.warning(
|
||||
"Detected Mooncake CPU/disk offloading pressure "
|
||||
"(NO_AVAILABLE_HANDLE); skipping future store "
|
||||
"batches for request %s until a later store batch succeeds",
|
||||
req_id,
|
||||
)
|
||||
|
||||
if not put_had_exception and not failed_indices:
|
||||
self._record_saved(req_meta, token_len)
|
||||
save_completed = True
|
||||
if self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared "
|
||||
"after a successful store batch"
|
||||
)
|
||||
|
||||
stored_events: list[BlockStored] = []
|
||||
if self.enable_kv_event and not put_had_exception:
|
||||
assert event_specs is not None
|
||||
# BlockStored is a logical-block event, while one block may map
|
||||
# to several Store shards. Emit once only after every missing
|
||||
# shard for that block succeeded. Shards that already existed
|
||||
# were removed before this mapping and are already satisfied.
|
||||
indices_by_event: dict[tuple[int, int, int, BlockHash], list[int]] = {}
|
||||
for index, event_spec in enumerate(event_specs):
|
||||
indices_by_event.setdefault(event_spec, []).append(index)
|
||||
|
||||
token_ids_end = token_ids_start + len(event_token_ids or ())
|
||||
for (
|
||||
s,
|
||||
end,
|
||||
g_idx,
|
||||
block_hash,
|
||||
), event_indices in indices_by_event.items():
|
||||
if any(index in failed_indices for index in event_indices):
|
||||
continue
|
||||
db = self.token_databases[g_idx]
|
||||
token_ids = (
|
||||
event_token_ids[s - token_ids_start : end - token_ids_start]
|
||||
if event_token_ids is not None
|
||||
and token_ids_start <= s
|
||||
and end <= token_ids_end
|
||||
else []
|
||||
)
|
||||
stored_events.append(
|
||||
BlockStored(
|
||||
block_hashes=[maybe_convert_block_hash(block_hash)],
|
||||
# Store filtering can separate adjacent request
|
||||
# blocks, so derive the predecessor from the request.
|
||||
parent_block_hash=(
|
||||
maybe_convert_block_hash(
|
||||
req_meta.block_hashes[s // db.hash_block_size - 1]
|
||||
)
|
||||
if s > 0
|
||||
else None
|
||||
),
|
||||
token_ids=token_ids,
|
||||
block_size=db.block_size,
|
||||
lora_id=None,
|
||||
medium="cpu",
|
||||
lora_name=None,
|
||||
group_idx=g_idx,
|
||||
)
|
||||
)
|
||||
|
||||
if self.enable_kv_event and stored_events:
|
||||
self.update_kv_event(stored_events)
|
||||
@@ -1269,6 +1313,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = load_mask_per_group[g_idx]
|
||||
chunks: list[tuple[int, int]] = []
|
||||
store_shard_ids: list[StoreShardId] = []
|
||||
for start, end, block_hash in db.process_tokens(
|
||||
token_len, req_meta.block_hashes, mask_num
|
||||
):
|
||||
@@ -1282,10 +1327,14 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
block_hash = req_meta.block_hashes[
|
||||
boundary_tokens // db.hash_block_size - 1
|
||||
]
|
||||
key_list.append(db.key_for(block_hash))
|
||||
chunks.append((start, end))
|
||||
g_addrs, g_sizes, g_block_ids = db.prepare_values(
|
||||
chunks, req_meta.block_ids[g_idx]
|
||||
for store_shard_id in db.store_layout.local_shard_ids:
|
||||
key_list.append(db.store_layout.key_for(store_shard_id, block_hash))
|
||||
chunks.append((start, end))
|
||||
store_shard_ids.append(store_shard_id)
|
||||
g_addrs, g_sizes, g_block_ids = db.store_layout.prepare_values(
|
||||
chunks,
|
||||
req_meta.block_ids[g_idx],
|
||||
store_shard_ids,
|
||||
)
|
||||
addr_list.extend(g_addrs)
|
||||
size_list.extend(g_sizes)
|
||||
@@ -1298,7 +1347,14 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
size_list_c = _rotate_list(size_list, rotation)
|
||||
block_id_list_c = _rotate_list(block_id_list, rotation)
|
||||
|
||||
load_batches = [(key_list_c, addr_list_c, size_list_c, block_id_list_c)]
|
||||
load_batches = [
|
||||
(
|
||||
key_list_c,
|
||||
addr_list_c,
|
||||
size_list_c,
|
||||
block_id_list_c,
|
||||
)
|
||||
]
|
||||
if self.usable_disk_offload_buffer_budget_bytes is not None:
|
||||
total_staging_bytes = sum(
|
||||
_estimate_disk_offload_staging_bytes(size) for size in size_list_c
|
||||
@@ -1340,7 +1396,12 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
block_id_offset:next_block_id_offset
|
||||
]
|
||||
load_batches.append(
|
||||
(batch_keys, batch_addrs, batch_sizes, batch_block_ids)
|
||||
(
|
||||
batch_keys,
|
||||
batch_addrs,
|
||||
batch_sizes,
|
||||
batch_block_ids,
|
||||
)
|
||||
)
|
||||
block_id_offset = next_block_id_offset
|
||||
|
||||
@@ -1348,7 +1409,12 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
current_batch_block_ids: list[int] = block_id_list_c
|
||||
batch_bytes = 0
|
||||
try:
|
||||
for batch_keys, batch_addrs, batch_sizes, batch_block_ids in load_batches:
|
||||
for (
|
||||
batch_keys,
|
||||
batch_addrs,
|
||||
batch_sizes,
|
||||
batch_block_ids,
|
||||
) in load_batches:
|
||||
current_batch_keys = batch_keys
|
||||
current_batch_block_ids = batch_block_ids
|
||||
batch_bytes = _sum_batch_bytes(batch_sizes)
|
||||
@@ -1607,11 +1673,9 @@ class MooncakeStoreWorker:
|
||||
retention_interval=kv_cache_config.prefix_cache_retention_interval,
|
||||
dcp_world_size=self.dcp_size,
|
||||
)
|
||||
# One ChunkedTokenDatabase per group; addresses populated in
|
||||
# register_kv_caches once the kv-cache layout is known. Each group's
|
||||
# key namespace is its TP shard id: ranks holding identical bytes
|
||||
# (MLA / shared GQA KV heads) share a namespace, TP-sharded Mamba
|
||||
# state gets one namespace per rank.
|
||||
self.store_tp_size, store_namespace, store_layout_cls = (
|
||||
self._select_store_layout(extra_config)
|
||||
)
|
||||
metadata = KeyMetadata(
|
||||
model_name=model_config.model.rstrip("/").split("/")[-1],
|
||||
tp_rank=self.tp_rank,
|
||||
@@ -1623,24 +1687,140 @@ class MooncakeStoreWorker:
|
||||
"cache_prefix", ""
|
||||
)
|
||||
),
|
||||
store_namespace=store_namespace,
|
||||
)
|
||||
self._group_tp_replication_factors: tuple[int, ...] = (
|
||||
self._compute_group_tp_replication_factors()
|
||||
)
|
||||
self.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
dataclasses.replace(
|
||||
metadata,
|
||||
group_id=g_idx,
|
||||
tp_rank=self.tp_rank // self._group_tp_replication_factors[g_idx],
|
||||
),
|
||||
g.kv_cache_spec.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
)
|
||||
for g_idx, g in enumerate(self._kv_cache_groups)
|
||||
]
|
||||
self.token_dbs = self._build_token_databases(metadata, store_layout_cls)
|
||||
self._init_lookup_key_prefixes()
|
||||
|
||||
def _supports_tp_sharded_store_layout(
|
||||
self,
|
||||
layout_cls: type[TPShardedStoreLayout] | None,
|
||||
extra_config: dict[str, Any],
|
||||
) -> bool:
|
||||
if (
|
||||
layout_cls is None
|
||||
or self.pcp_size != 1
|
||||
or self.dcp_size != 1
|
||||
or len(self._kv_cache_groups) != 1
|
||||
):
|
||||
return False
|
||||
|
||||
# Subclasses may use different sharding or cache-lifetime semantics.
|
||||
return (
|
||||
type(self._kv_cache_groups[0].kv_cache_spec) is FullAttentionSpec
|
||||
and str(extra_config.get("enable_cross_layers_blocks", "False")).lower()
|
||||
!= "true"
|
||||
)
|
||||
|
||||
def _select_store_layout(
|
||||
self, extra_config: dict[str, Any]
|
||||
) -> tuple[int | None, str, type[TPShardedStoreLayout] | None]:
|
||||
"""Select the opt-in TP layout and its Store namespace."""
|
||||
lcm_store_tp_enabled = extra_config.get("enable_store_tp_lcm") is True
|
||||
store_tp_requested = (
|
||||
lcm_store_tp_enabled or extra_config.get("store_tp_size") is not None
|
||||
)
|
||||
if not store_tp_requested:
|
||||
return None, "", None
|
||||
|
||||
requested_store_tp_size = resolve_store_tp_size(extra_config)
|
||||
cache_layout = self.cache_config.get_resolved_kv_cache_layout()
|
||||
layout_cls: type[TPShardedStoreLayout] | None = {
|
||||
KVCacheLayout.LBHNC: LBHNCStoreLayout,
|
||||
KVCacheLayout.LBNHC: LBNHCStoreLayout,
|
||||
}.get(cache_layout)
|
||||
|
||||
if (
|
||||
requested_store_tp_size is not None
|
||||
and requested_store_tp_size >= self.tp_size
|
||||
and requested_store_tp_size % self.tp_size == 0
|
||||
and self._supports_tp_sharded_store_layout(layout_cls, extra_config)
|
||||
):
|
||||
assert layout_cls is not None
|
||||
if self.num_kv_head % requested_store_tp_size == 0:
|
||||
if layout_cls is LBNHCStoreLayout:
|
||||
logger.warning_once(
|
||||
"Mooncake Store TP sharding is using the LBNHC (NHC) "
|
||||
"KV cache layout, which creates many transfer segments "
|
||||
"and may significantly reduce PUT/GET performance. Use "
|
||||
"LBHNC (HNC) when supported by the attention backend."
|
||||
)
|
||||
return (
|
||||
requested_store_tp_size,
|
||||
layout_cls.shared_namespace(requested_store_tp_size, self.pp_size),
|
||||
layout_cls,
|
||||
)
|
||||
if self.num_kv_head == 1:
|
||||
logger.info(
|
||||
"Mooncake heterogeneous-TP store sharing uses the replicated "
|
||||
"MQA layout for store_tp_size=%d",
|
||||
requested_store_tp_size,
|
||||
)
|
||||
return (
|
||||
None,
|
||||
f"@store_pp:{self.pp_size}@store_format:tp_shared_mqa",
|
||||
None,
|
||||
)
|
||||
|
||||
store_namespace = (
|
||||
f"@store_pp:{self.pp_size}@store_format:"
|
||||
f"rank_local_tp{self.tp_size}_layout_{cache_layout.name}"
|
||||
)
|
||||
requested_topology = (
|
||||
extra_config.get("prefill_tp_sizes")
|
||||
if lcm_store_tp_enabled
|
||||
else extra_config.get("store_tp_size")
|
||||
)
|
||||
logger.warning(
|
||||
"Mooncake heterogeneous-TP store sharing is disabled for "
|
||||
"Store TP configuration %r with KV layout %s; using a "
|
||||
"compatibility-namespaced rank-local store layout",
|
||||
requested_topology,
|
||||
cache_layout,
|
||||
)
|
||||
return None, store_namespace, None
|
||||
|
||||
def _build_token_databases(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
layout_cls: type[TPShardedStoreLayout] | None,
|
||||
) -> list[ChunkedTokenDatabase]:
|
||||
"""Construct token databases and their Store layouts."""
|
||||
token_dbs: list[ChunkedTokenDatabase] = []
|
||||
for group_idx, group in enumerate(self._kv_cache_groups):
|
||||
group_tp_rank = self.tp_rank
|
||||
if layout_cls is None:
|
||||
group_tp_rank //= self._group_tp_replication_factors[group_idx]
|
||||
group_metadata = dataclasses.replace(
|
||||
metadata,
|
||||
group_id=group_idx,
|
||||
tp_rank=group_tp_rank,
|
||||
)
|
||||
store_layout: TPShardedStoreLayout | None = None
|
||||
if layout_cls is not None:
|
||||
assert self.store_tp_size is not None
|
||||
store_layout = layout_cls(
|
||||
group_metadata,
|
||||
group.kv_cache_spec.block_size,
|
||||
self.hash_block_size,
|
||||
local_tp_size=self.tp_size,
|
||||
store_tp_size=self.store_tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
num_kv_heads=self.num_kv_head,
|
||||
)
|
||||
token_dbs.append(
|
||||
ChunkedTokenDatabase(
|
||||
group_metadata,
|
||||
group.kv_cache_spec.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
store_layout=store_layout,
|
||||
)
|
||||
)
|
||||
return token_dbs
|
||||
|
||||
def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int:
|
||||
if self.dcp_size > 1:
|
||||
return 1
|
||||
@@ -1689,17 +1869,8 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
|
||||
self._lookup_key_prefixes = tuple(
|
||||
tuple(
|
||||
PoolKey.build_prefix(
|
||||
db.metadata,
|
||||
tp_rank=tp_rank,
|
||||
pcp_rank=pcp_rank,
|
||||
dcp_rank=dcp_rank,
|
||||
pp_rank=pp_rank,
|
||||
)
|
||||
for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces(
|
||||
self._group_tp_replication_factors[g_idx]
|
||||
)
|
||||
db.store_layout.lookup_key_prefixes(
|
||||
rank_namespaces(self._group_tp_replication_factors[g_idx])
|
||||
)
|
||||
for g_idx, db in enumerate(self.token_dbs)
|
||||
)
|
||||
@@ -1719,12 +1890,11 @@ class MooncakeStoreWorker:
|
||||
self.num_blocks = self.cache_config.num_gpu_blocks
|
||||
|
||||
seen_storage_ptrs: set[int] = set()
|
||||
seen_region_ptrs: set[int] = set()
|
||||
addrs: list[int] = []
|
||||
block_lens: list[int] = []
|
||||
cache_tensors: list[torch.Tensor] = []
|
||||
|
||||
for cache in kv_caches.values():
|
||||
cache = group_kernel_blocks(cache, self.num_blocks)
|
||||
cache_tensors.append(cache)
|
||||
cache_storage = cache.untyped_storage()
|
||||
base_addr = cache_storage.data_ptr()
|
||||
region_len = cache_storage.nbytes()
|
||||
@@ -1740,48 +1910,15 @@ class MooncakeStoreWorker:
|
||||
ret,
|
||||
)
|
||||
|
||||
if not is_non_overlapping_and_dense(cache[0]):
|
||||
# A block is scattered across per-head regions; each region's
|
||||
# blocks are contiguous.
|
||||
for head_idx in range(cache.shape[1]):
|
||||
head_cache = cache[:, head_idx]
|
||||
assert is_non_overlapping_and_dense(head_cache[0])
|
||||
region_addr = head_cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
addrs.append(region_addr)
|
||||
block_lens.append(head_cache.stride(0) * head_cache.element_size())
|
||||
elif cache.stride(0) * cache.element_size() * self.num_blocks == region_len:
|
||||
# The block stride spans the whole per-block window
|
||||
# (block-outermost and packed layouts, and single-layer
|
||||
# tensors), which may hold other layers' pages at higher
|
||||
# offsets. Register the storage once as one whole-window
|
||||
# region; per-layer regions would copy overlapping windows and
|
||||
# run past the storage for offset layers.
|
||||
if base_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(base_addr)
|
||||
addrs.append(base_addr)
|
||||
block_lens.append(region_len // self.num_blocks)
|
||||
else:
|
||||
region_addr = cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
addrs.append(region_addr)
|
||||
block_lens.append(cache.stride(0) * cache.element_size())
|
||||
|
||||
logger.info(
|
||||
"Registered KV caches: num_groups=%d, num_segments=%d, num_blocks=%d",
|
||||
"Registered KV caches: num_groups=%d, num_tensors=%d, num_blocks=%d",
|
||||
len(self.token_dbs),
|
||||
len(addrs),
|
||||
len(cache_tensors),
|
||||
self.num_blocks,
|
||||
)
|
||||
|
||||
for db in self.token_dbs:
|
||||
db.set_kv_caches_base_addr(addrs)
|
||||
db.set_block_len(block_lens)
|
||||
db.store_layout.register_kv_caches(cache_tensors, self.num_blocks)
|
||||
|
||||
# Start transfer threads
|
||||
if self.can_put:
|
||||
@@ -2128,6 +2265,10 @@ class MooncakeStoreWorker:
|
||||
return self.kv_send_thread.get_kv_events()
|
||||
return []
|
||||
|
||||
@property
|
||||
def group_tp_replication_factors(self) -> tuple[int, ...]:
|
||||
return self._group_tp_replication_factors
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release the MooncakeDistributedStore handle on teardown.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user