[KVConnector][Mooncake] Wire reset_cache cascade end-to-end (#42694)

Signed-off-by: aoshen524 <aoshen524@gmail.com>
Signed-off-by: Ao Shen <aoshen@inferact.ai>
Co-authored-by: aoshen524 <aoshen524@gmail.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-05-26 20:52:35 -07:00
committed by GitHub
co-authored by aoshen524 Claude Opus 4.7
parent 0fa3114ae1
commit 1fc2cee50a
8 changed files with 517 additions and 14 deletions
@@ -200,6 +200,23 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
def reset_cache(self) -> bool | None:
"""Reset the external Mooncake store on prefix-cache reset.
Drains the worker send queue, then runs ``remove_all`` on the
Mooncake master. Caller must first pause generation (e.g.
``pause_generation``) so no new puts are enqueued during drain.
Returns True on ack, False on failure, None for the worker role.
"""
if self.role == KVConnectorRole.SCHEDULER:
assert self.connector_scheduler is not None
# Clear local references to keys we're about to wipe.
self.connector_scheduler.load_specs.clear()
self._kv_cache_events = None
return self.connector_scheduler.reset_store()
return None
def update_connector_output(self, connector_output: KVConnectorOutput):
kv_cache_events = connector_output.kv_cache_events
if not kv_cache_events or not isinstance(
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Wire-format constants for the LookupKey ZMQ admin channel.
This is the single source of truth shared by ``LookupKeyClient`` and
``LookupKeyServer`` on the scheduler<->worker rank-0 admin channel.
Wire format (REQ/REP over IPC):
Request: [msg_type: bytes] [payload_frames...]
msg_type == LOOKUP_MSG:
frame 1: token_len (u32 big-endian, 4 bytes)
frame 2..n: msgpack-encoded list[str] of block-hash hex digests
Response: [hit_count: u32 big-endian, 4 bytes]
msg_type == RESET_MSG:
(no payload frames)
Response: [RESP_OK] or [RESP_ERR]
The first frame of every request is a named bytes tag (not a numeric
sentinel that aliases the data field) so the protocol stays
self-describing and extensible: adding new admin commands requires
only a new tag and a new dispatch branch.
Mirrors the named-tag convention used by the NIXL connector (see
``vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py``).
"""
# Request message-type tags. Frame 0 of every request.
LOOKUP_MSG: bytes = b"lookup"
RESET_MSG: bytes = b"reset"
# Single-byte response status codes for admin commands.
RESP_OK: bytes = b"\x01"
RESP_ERR: bytes = b"\x00"
@@ -371,3 +371,30 @@ class MooncakeStoreScheduler:
request.request_id,
)
return delay_free_blocks, None
def reset_store(self) -> bool:
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
Routes through the existing LookupKey ZMQ admin channel to worker
rank 0, which owns the ``MooncakeDistributedStore`` handle.
Ordering assumption: caller (typically
``Scheduler.reset_connector_cache``, invoked via
``reset_prefix_cache(reset_connector=True)``) MUST ensure no
in-flight Mooncake lookups or transfers. For RL workflows this is
satisfied at the step boundary after weight updates and rollout
drain. Violating this can allow stale KV to be served on the next
request, defeating the hard-reset guarantee.
Returns True on ACK from worker, False on NACK or RPC error.
"""
try:
ok = self.client.reset()
if ok:
logger.info("Mooncake store reset via remove_all succeeded.")
else:
logger.warning("Mooncake store reset returned NACK from worker.")
return ok
except Exception as e:
logger.error("Mooncake reset_store RPC failed: %s", e)
return False
@@ -50,6 +50,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
PoolKey,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import ( # noqa: E501
LOOKUP_MSG,
RESET_MSG,
RESP_ERR,
RESP_OK,
)
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip, make_zmq_socket
from vllm.v1.core.kv_cache_utils import (
@@ -1424,7 +1430,14 @@ class MooncakeStoreWorker:
class LookupKeyServer:
"""ZMQ server on worker rank 0 for handling prefix lookup queries."""
"""ZMQ server on worker rank 0 for the LookupKey admin channel.
Handles two request types, tagged at frame 0:
- ``LOOKUP_MSG``: prefix-cache hit query, returns hit count.
- ``RESET_MSG``: drains the send thread queue, then runs
``store.remove_all(force=True)``. Caller must have paused the
scheduler first.
"""
def __init__(
self,
@@ -1450,13 +1463,37 @@ class LookupKeyServer:
def process_request():
while self.running:
all_frames = self.socket.recv_multipart(copy=False)
token_len = int.from_bytes(all_frames[0], byteorder="big")
hash_frames = all_frames[1:]
hashes_str = self.decoder.decode(hash_frames)
block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str]
result = self.store_worker.lookup(token_len, block_hashes)
response = result.to_bytes(4, "big")
self.socket.send(response)
msg_type = bytes(all_frames[0])
if msg_type == LOOKUP_MSG:
token_len = int.from_bytes(all_frames[1], byteorder="big")
hash_frames = all_frames[2:]
hashes_str = self.decoder.decode(hash_frames)
block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str]
result = self.store_worker.lookup(token_len, block_hashes)
self.socket.send(result.to_bytes(4, "big"))
elif msg_type == RESET_MSG:
try:
# Drain in-flight puts before wiping the master;
# otherwise stale puts can repopulate it post-reset.
# Safe across HMA: store.remove_all wipes the underlying
# flat key space, clearing every (group_id, hash) entry.
if self.store_worker.kv_send_thread is not None:
self.store_worker.kv_send_thread.request_queue.join()
self.store_worker.store.remove_all(force=True)
logger.info("Mooncake store reset via remove_all succeeded.")
self.socket.send(RESP_OK)
except Exception as e:
logger.error("Mooncake remove_all failed: %s", e)
self.socket.send(RESP_ERR)
else:
logger.warning(
"LookupKeyServer received unknown msg_type: %r",
msg_type,
)
self.socket.send(RESP_ERR)
self.thread = threading.Thread(target=process_request, daemon=True)
self.thread.start()
@@ -1473,7 +1510,12 @@ class LookupKeyServer:
class LookupKeyClient:
"""ZMQ client for querying prefix cache hits from worker."""
"""ZMQ client for the LookupKey admin channel.
Routes both prefix-cache lookups and admin commands (currently:
``reset``) to ``LookupKeyServer`` on worker rank 0. The first frame
of every request is a named tag from ``protocol.py``.
"""
def __init__(self, vllm_config: VllmConfig):
self.encoder = MsgpackEncoder()
@@ -1490,12 +1532,24 @@ class LookupKeyClient:
hash_strs = [h.hex() for h in block_hashes]
hash_frames = self.encoder.encode(hash_strs)
token_len_bytes = token_len.to_bytes(4, byteorder="big")
all_frames = [token_len_bytes] + list(hash_frames)
all_frames = [LOOKUP_MSG, token_len_bytes] + list(hash_frames)
self.socket.send_multipart(all_frames, copy=False)
resp = self.socket.recv()
result = int.from_bytes(resp, "big")
return result
def reset(self) -> bool:
"""Trigger ``store.remove_all(force=True)`` on worker rank 0.
Ordering assumption: caller MUST ensure no in-flight Mooncake
lookups or transfers when invoking reset. In RL workflows this
holds naturally at the step boundary after weight updates and
rollout drain. Returns True on ACK, False on NACK.
"""
self.socket.send(RESET_MSG)
resp = self.socket.recv()
return bytes(resp) == RESP_OK
def close(self):
self.socket.close(linger=0)