[KV Connector][Mooncake] Add store group semantics (#44956)

Signed-off-by: Schatten <czhengt@qq.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Schatten
2026-08-05 08:25:38 +00:00
committed by GitHub
co-authored by OpenAI Codex Yifan Qiao
parent f5cd862dbd
commit fd859d2e06
2 changed files with 531 additions and 3 deletions
@@ -89,6 +89,22 @@ def _rotate_list(values: list[_T], offset: int) -> list[_T]:
return values[offset:] + values[:offset]
def _replicate_config_supports_group_ids(
replicate_config_cls: type[Any],
replicate_config: Any,
) -> bool:
if hasattr(replicate_config_cls, "group_ids"):
return True
return hasattr(replicate_config, "group_ids")
def _make_mooncake_group_id(metadata: KeyMetadata, chunk_hash: str) -> str:
# Mooncake group ids describe the lifecycle unit. For vLLM, that unit is
# a prefix chunk, so shard dimensions stay only in the object key.
prefix = f"{metadata.cache_prefix}@" if metadata.cache_prefix else ""
return f"vllm-mooncake-store:{prefix}{metadata.model_name}@{chunk_hash}"
# Mirrors FileStorageConfig::local_buffer_size in Mooncake C++.
DEFAULT_MOONCAKE_DISK_STAGING_BUFFER_BYTES = 1280 * 1024 * 1024
@@ -459,6 +475,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
ready_event: threading.Event,
enable_kv_event: bool = False,
replicate_config: Any = None,
enable_group_semantics: bool = False,
supports_group_ids: bool = False,
record_operation: Callable[..., None] | None = None,
):
super().__init__(
@@ -479,6 +497,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
# Caller always passes a non-None ReplicateConfig — see
# MooncakeStoreWorker.__init__ where store_replicate_config is built.
self.replicate_config = replicate_config
self.enable_group_semantics = enable_group_semantics
self.supports_group_ids = supports_group_ids
# Pause store requests when CPU/disk offloading is under pressure.
self._store_pressure_active = False
@@ -567,6 +587,9 @@ class KVCacheStoreSendingThread(KVTransferThread):
keys: list[str] = []
addrs: list[list[int]] = []
sizes: list[list[int]] = []
group_ids: list[str] | None = (
[] if self.enable_group_semantics and self.supports_group_ids else None
)
saved = self._saved_offset.get(req_meta.req_id, 0)
for g_idx, db in enumerate(self.token_databases):
group_blocks = req_meta.block_ids[g_idx]
@@ -603,9 +626,17 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
continue
addr, size = db.prepare_value_for_block(block_id)
keys.append(db.key_for(key_hash))
key = db.key_for(key_hash)
keys.append(key)
addrs.append(addr)
sizes.append(size)
if group_ids is not None:
group_ids.append(
_make_mooncake_group_id(
db.metadata,
key.rsplit("@", 1)[-1],
)
)
if not keys:
return True
@@ -633,9 +664,14 @@ class KVCacheStoreSendingThread(KVTransferThread):
keys = [keys[i] for i in missing]
addrs = [addrs[i] for i in missing]
sizes = [sizes[i] for i in missing]
if group_ids is not None:
group_ids = [group_ids[i] for i in missing]
if req_meta.current_event is not None:
# Fence the CoW block copy enqueued earlier this step.
req_meta.current_event.synchronize()
if group_ids is not None:
assert len(group_ids) == len(keys)
self.replicate_config.group_ids = group_ids
batch_bytes = _sum_batch_bytes(sizes)
put_start = time.perf_counter()
try:
@@ -797,6 +833,18 @@ class KVCacheStoreSendingThread(KVTransferThread):
]
group_indices = [group_indices[i] for i in missing_indices]
group_ids = (
[
_make_mooncake_group_id(
self.token_databases[g_idx].metadata,
key.rsplit("@", 1)[-1],
)
for key, g_idx in zip(keys, group_indices, strict=True)
]
if self.enable_group_semantics and self.supports_group_ids
else None
)
logger.debug(
"Storing KV cache for %d blocks (groups=%s) for request %s",
len(keys),
@@ -855,6 +903,10 @@ class KVCacheStoreSendingThread(KVTransferThread):
if current_event is not None:
current_event.synchronize()
if group_ids is not None:
assert len(group_ids) == len(keys)
self.replicate_config.group_ids = group_ids
batch_bytes = _sum_batch_bytes(sizes)
put_start = time.perf_counter()
try:
@@ -1210,6 +1262,19 @@ class MooncakeStoreWorker:
preferred_segment = rdma_utils.get_configured_preferred_segment(extra_config)
self.preferred_segment = preferred_segment
self.store_replicate_config = ReplicateConfig()
self.enable_group_semantics = (
str(extra_config.get("enable_group_semantics", "False")).strip().lower()
== "true"
)
self._supports_group_ids = _replicate_config_supports_group_ids(
ReplicateConfig, self.store_replicate_config
)
if self.enable_group_semantics and not self._supports_group_ids:
logger.warning(
"Mooncake group semantics is enabled, but the installed "
"Mooncake package does not support ReplicateConfig.group_ids. "
"Falling back to the existing batch_put_from_multi_buffers path."
)
if preferred_segment is not None:
self.store_replicate_config.preferred_segment = preferred_segment
@@ -1503,6 +1568,8 @@ class MooncakeStoreWorker:
ready_event_sending,
self.enable_kv_events,
self.store_replicate_config,
enable_group_semantics=self.enable_group_semantics,
supports_group_ids=self._supports_group_ids,
record_operation=self._record_kv_connector_operation,
)
self.kv_send_thread.start()