[KVConnector][Mooncake] Wire reset_cache cascade end-to-end (#42694)
Signed-off-by: aoshen524 <aoshen524@gmail.com> Signed-off-by: Ao Shen <aoshen@inferact.ai> Co-authored-by: aoshen524 <aoshen524@gmail.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen524
Claude Opus 4.7
parent
0fa3114ae1
commit
1fc2cee50a
@@ -200,6 +200,23 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
"""Reset the external Mooncake store on prefix-cache reset.
|
||||
|
||||
Drains the worker send queue, then runs ``remove_all`` on the
|
||||
Mooncake master. Caller must first pause generation (e.g.
|
||||
``pause_generation``) so no new puts are enqueued during drain.
|
||||
|
||||
Returns True on ack, False on failure, None for the worker role.
|
||||
"""
|
||||
if self.role == KVConnectorRole.SCHEDULER:
|
||||
assert self.connector_scheduler is not None
|
||||
# Clear local references to keys we're about to wipe.
|
||||
self.connector_scheduler.load_specs.clear()
|
||||
self._kv_cache_events = None
|
||||
return self.connector_scheduler.reset_store()
|
||||
return None
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
kv_cache_events = connector_output.kv_cache_events
|
||||
if not kv_cache_events or not isinstance(
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Wire-format constants for the LookupKey ZMQ admin channel.
|
||||
|
||||
This is the single source of truth shared by ``LookupKeyClient`` and
|
||||
``LookupKeyServer`` on the scheduler<->worker rank-0 admin channel.
|
||||
|
||||
Wire format (REQ/REP over IPC):
|
||||
|
||||
Request: [msg_type: bytes] [payload_frames...]
|
||||
|
||||
msg_type == LOOKUP_MSG:
|
||||
frame 1: token_len (u32 big-endian, 4 bytes)
|
||||
frame 2..n: msgpack-encoded list[str] of block-hash hex digests
|
||||
Response: [hit_count: u32 big-endian, 4 bytes]
|
||||
|
||||
msg_type == RESET_MSG:
|
||||
(no payload frames)
|
||||
Response: [RESP_OK] or [RESP_ERR]
|
||||
|
||||
The first frame of every request is a named bytes tag (not a numeric
|
||||
sentinel that aliases the data field) so the protocol stays
|
||||
self-describing and extensible: adding new admin commands requires
|
||||
only a new tag and a new dispatch branch.
|
||||
|
||||
Mirrors the named-tag convention used by the NIXL connector (see
|
||||
``vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py``).
|
||||
"""
|
||||
|
||||
# Request message-type tags. Frame 0 of every request.
|
||||
LOOKUP_MSG: bytes = b"lookup"
|
||||
RESET_MSG: bytes = b"reset"
|
||||
|
||||
# Single-byte response status codes for admin commands.
|
||||
RESP_OK: bytes = b"\x01"
|
||||
RESP_ERR: bytes = b"\x00"
|
||||
@@ -371,3 +371,30 @@ class MooncakeStoreScheduler:
|
||||
request.request_id,
|
||||
)
|
||||
return delay_free_blocks, None
|
||||
|
||||
def reset_store(self) -> bool:
|
||||
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
|
||||
|
||||
Routes through the existing LookupKey ZMQ admin channel to worker
|
||||
rank 0, which owns the ``MooncakeDistributedStore`` handle.
|
||||
|
||||
Ordering assumption: caller (typically
|
||||
``Scheduler.reset_connector_cache``, invoked via
|
||||
``reset_prefix_cache(reset_connector=True)``) MUST ensure no
|
||||
in-flight Mooncake lookups or transfers. For RL workflows this is
|
||||
satisfied at the step boundary after weight updates and rollout
|
||||
drain. Violating this can allow stale KV to be served on the next
|
||||
request, defeating the hard-reset guarantee.
|
||||
|
||||
Returns True on ACK from worker, False on NACK or RPC error.
|
||||
"""
|
||||
try:
|
||||
ok = self.client.reset()
|
||||
if ok:
|
||||
logger.info("Mooncake store reset via remove_all succeeded.")
|
||||
else:
|
||||
logger.warning("Mooncake store reset returned NACK from worker.")
|
||||
return ok
|
||||
except Exception as e:
|
||||
logger.error("Mooncake reset_store RPC failed: %s", e)
|
||||
return False
|
||||
|
||||
@@ -50,6 +50,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
PoolKey,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import ( # noqa: E501
|
||||
LOOKUP_MSG,
|
||||
RESET_MSG,
|
||||
RESP_ERR,
|
||||
RESP_OK,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import get_ip, make_zmq_socket
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
@@ -1424,7 +1430,14 @@ class MooncakeStoreWorker:
|
||||
|
||||
|
||||
class LookupKeyServer:
|
||||
"""ZMQ server on worker rank 0 for handling prefix lookup queries."""
|
||||
"""ZMQ server on worker rank 0 for the LookupKey admin channel.
|
||||
|
||||
Handles two request types, tagged at frame 0:
|
||||
- ``LOOKUP_MSG``: prefix-cache hit query, returns hit count.
|
||||
- ``RESET_MSG``: drains the send thread queue, then runs
|
||||
``store.remove_all(force=True)``. Caller must have paused the
|
||||
scheduler first.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1450,13 +1463,37 @@ class LookupKeyServer:
|
||||
def process_request():
|
||||
while self.running:
|
||||
all_frames = self.socket.recv_multipart(copy=False)
|
||||
token_len = int.from_bytes(all_frames[0], byteorder="big")
|
||||
hash_frames = all_frames[1:]
|
||||
hashes_str = self.decoder.decode(hash_frames)
|
||||
block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str]
|
||||
result = self.store_worker.lookup(token_len, block_hashes)
|
||||
response = result.to_bytes(4, "big")
|
||||
self.socket.send(response)
|
||||
msg_type = bytes(all_frames[0])
|
||||
|
||||
if msg_type == LOOKUP_MSG:
|
||||
token_len = int.from_bytes(all_frames[1], byteorder="big")
|
||||
hash_frames = all_frames[2:]
|
||||
hashes_str = self.decoder.decode(hash_frames)
|
||||
block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str]
|
||||
result = self.store_worker.lookup(token_len, block_hashes)
|
||||
self.socket.send(result.to_bytes(4, "big"))
|
||||
|
||||
elif msg_type == RESET_MSG:
|
||||
try:
|
||||
# Drain in-flight puts before wiping the master;
|
||||
# otherwise stale puts can repopulate it post-reset.
|
||||
# Safe across HMA: store.remove_all wipes the underlying
|
||||
# flat key space, clearing every (group_id, hash) entry.
|
||||
if self.store_worker.kv_send_thread is not None:
|
||||
self.store_worker.kv_send_thread.request_queue.join()
|
||||
self.store_worker.store.remove_all(force=True)
|
||||
logger.info("Mooncake store reset via remove_all succeeded.")
|
||||
self.socket.send(RESP_OK)
|
||||
except Exception as e:
|
||||
logger.error("Mooncake remove_all failed: %s", e)
|
||||
self.socket.send(RESP_ERR)
|
||||
|
||||
else:
|
||||
logger.warning(
|
||||
"LookupKeyServer received unknown msg_type: %r",
|
||||
msg_type,
|
||||
)
|
||||
self.socket.send(RESP_ERR)
|
||||
|
||||
self.thread = threading.Thread(target=process_request, daemon=True)
|
||||
self.thread.start()
|
||||
@@ -1473,7 +1510,12 @@ class LookupKeyServer:
|
||||
|
||||
|
||||
class LookupKeyClient:
|
||||
"""ZMQ client for querying prefix cache hits from worker."""
|
||||
"""ZMQ client for the LookupKey admin channel.
|
||||
|
||||
Routes both prefix-cache lookups and admin commands (currently:
|
||||
``reset``) to ``LookupKeyServer`` on worker rank 0. The first frame
|
||||
of every request is a named tag from ``protocol.py``.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
self.encoder = MsgpackEncoder()
|
||||
@@ -1490,12 +1532,24 @@ class LookupKeyClient:
|
||||
hash_strs = [h.hex() for h in block_hashes]
|
||||
hash_frames = self.encoder.encode(hash_strs)
|
||||
token_len_bytes = token_len.to_bytes(4, byteorder="big")
|
||||
all_frames = [token_len_bytes] + list(hash_frames)
|
||||
all_frames = [LOOKUP_MSG, token_len_bytes] + list(hash_frames)
|
||||
self.socket.send_multipart(all_frames, copy=False)
|
||||
resp = self.socket.recv()
|
||||
result = int.from_bytes(resp, "big")
|
||||
return result
|
||||
|
||||
def reset(self) -> bool:
|
||||
"""Trigger ``store.remove_all(force=True)`` on worker rank 0.
|
||||
|
||||
Ordering assumption: caller MUST ensure no in-flight Mooncake
|
||||
lookups or transfers when invoking reset. In RL workflows this
|
||||
holds naturally at the step boundary after weight updates and
|
||||
rollout drain. Returns True on ACK, False on NACK.
|
||||
"""
|
||||
self.socket.send(RESET_MSG)
|
||||
resp = self.socket.recv()
|
||||
return bytes(resp) == RESP_OK
|
||||
|
||||
def close(self):
|
||||
self.socket.close(linger=0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user