Secondary tier implementation for PD disaggregation (#42285)
Signed-off-by: Liran Schour <lirans@il.ibm.com> Signed-off-by: liranschour <liranschour@users.noreply.github.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Or Ozeri
Or Ozeri
parent
b5c9e1ac33
commit
bec232a914
@@ -66,6 +66,12 @@ SecondaryTierFactory.register_tier(
|
||||
"FileSystemTierManager",
|
||||
)
|
||||
|
||||
SecondaryTierFactory.register_tier(
|
||||
"p2p",
|
||||
"vllm.v1.kv_offload.tiering.p2p.manager",
|
||||
"P2PSecondaryTierManager",
|
||||
)
|
||||
|
||||
SecondaryTierFactory.register_tier(
|
||||
"obj",
|
||||
"vllm.v1.kv_offload.tiering.obj.manager",
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.base import (
|
||||
ControlConnection,
|
||||
ControlTransport,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
|
||||
ZmqConnection,
|
||||
ZmqTransport,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ControlConnection",
|
||||
"ControlTransport",
|
||||
"ZmqConnection",
|
||||
"ZmqTransport",
|
||||
]
|
||||
@@ -0,0 +1,166 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base classes for the P2P control-plane transport.
|
||||
|
||||
The control plane handles peer discovery, connection lifecycle, and
|
||||
message routing. It is message-content agnostic — it moves opaque
|
||||
dicts between peers without interpreting them.
|
||||
|
||||
Architecture
|
||||
------------
|
||||
|
||||
ControlTransport (one per node)
|
||||
├── listen for inbound connections
|
||||
├── connect() to outbound peers
|
||||
└── poll() → new ControlConnections
|
||||
|
||||
ControlConnection (one per peer)
|
||||
├── send(msg) — enqueue a message to the peer
|
||||
├── recv() — drain buffered inbound messages
|
||||
├── mark_dead() — signal that the peer is gone
|
||||
└── close() — tear down the connection
|
||||
|
||||
Threading model: all I/O is driven by the caller invoking poll().
|
||||
No background threads. poll() must be called periodically to:
|
||||
- receive messages (buffered per-connection)
|
||||
- accept new inbound peers
|
||||
- detect disconnections
|
||||
|
||||
Implementor contracts
|
||||
---------------------
|
||||
|
||||
- ControlConnection.send() must not block. Messages are serialized
|
||||
and queued for the next I/O pass.
|
||||
- ControlConnection.recv() returns all messages received since the
|
||||
last call (may be empty). Messages are dicts (already deserialized).
|
||||
- ControlConnection.alive returns False after mark_dead() or close().
|
||||
- ControlTransport.poll() returns newly accepted connections only —
|
||||
not previously returned ones. Each connection appears exactly once.
|
||||
- ControlTransport.connect() creates an outbound connection to a peer
|
||||
identified by peer_id (format: "host:port"). Raises on failure.
|
||||
- Messages may arrive from unknown peers (new inbound connections).
|
||||
The transport creates a ControlConnection and returns it from poll()
|
||||
with the first message(s) already in its recv() buffer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
class ControlConnection(ABC):
|
||||
"""Bidirectional message channel to a single remote peer.
|
||||
|
||||
Lifecycle:
|
||||
1. Created by ControlTransport (connect() or poll())
|
||||
2. Used for send/recv by sessions
|
||||
3. Marked dead on disconnect (mark_dead())
|
||||
4. Cleaned up with close()
|
||||
|
||||
Once mark_dead() is called, alive becomes False and the owning
|
||||
session should stop using this connection. close() releases
|
||||
underlying resources (sockets, monitors).
|
||||
"""
|
||||
|
||||
def __init__(self, peer_id: str) -> None:
|
||||
self.peer_id = peer_id
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def alive(self) -> bool:
|
||||
"""True if the connection is usable. False after mark_dead/close."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def send(self, msg: dict) -> None:
|
||||
"""Enqueue a message for delivery to the peer.
|
||||
|
||||
Must not block. Serialization happens internally.
|
||||
Raises on closed connection.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def recv(self) -> Sequence[dict]:
|
||||
"""Drain and return all messages received since the last call.
|
||||
|
||||
Returns an empty sequence if no messages are pending.
|
||||
The returned sequence is read-only — callers must not mutate it.
|
||||
Messages are dicts deserialized from the wire format.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def mark_dead(self) -> None:
|
||||
"""Mark this connection as dead (peer disconnected).
|
||||
|
||||
After this call, alive returns False. The session should
|
||||
stop using this connection and the transport will clean it up.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Release all resources (sockets, monitors).
|
||||
|
||||
Idempotent. After close(), alive returns False.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class ControlTransport(ABC):
|
||||
"""Manages peer connections and drives all control-plane I/O.
|
||||
|
||||
Owns the listening socket and all active connections.
|
||||
The caller must invoke poll() periodically to process I/O.
|
||||
|
||||
Lifecycle:
|
||||
1. Constructed with a local identity and listen address
|
||||
2. connect() to reach remote peers
|
||||
3. poll() to accept inbound peers and process messages
|
||||
4. close() to shut down
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def connect(self, peer_id: str) -> ControlConnection:
|
||||
"""Create an outbound connection to a remote peer.
|
||||
|
||||
Args:
|
||||
peer_id: Remote peer identity (format: "host:port").
|
||||
|
||||
Returns:
|
||||
A new ControlConnection ready for send/recv.
|
||||
|
||||
The connection's send queue is live immediately — messages
|
||||
sent before the remote peer's poll() will be buffered.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def poll(self) -> Sequence[ControlConnection]:
|
||||
"""Process all pending I/O and return newly accepted connections.
|
||||
|
||||
This is the main I/O driver. Each call:
|
||||
- Receives messages from all connected peers (buffered in
|
||||
each connection's recv() queue)
|
||||
- Accepts new inbound peers and creates connections for them
|
||||
(first message already in recv() buffer)
|
||||
- Detects disconnections and marks connections dead
|
||||
|
||||
Returns:
|
||||
Newly accepted inbound connections (not previously returned).
|
||||
The returned sequence is read-only — callers must not mutate
|
||||
it. The caller is responsible for creating sessions for
|
||||
these connections.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Shut down the transport and all connections.
|
||||
|
||||
Closes the listening socket and all active connections.
|
||||
Idempotent.
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,307 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ZMQ-based transport layer for P2P KV cache sharing.
|
||||
|
||||
Provides ZmqConnection (per-peer messaging) and ZmqTransport (connection
|
||||
management). Message-content agnostic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
import zmq.utils.monitor
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.base import (
|
||||
ControlConnection,
|
||||
ControlTransport,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HEARTBEAT_IVL_MS = 2000
|
||||
_HEARTBEAT_TIMEOUT_MS = 10000
|
||||
_HEARTBEAT_TTL_MS = 10000
|
||||
|
||||
# Shared sentinels returned when there is nothing to report.
|
||||
_EMPTY_INBOX: tuple[dict, ...] = ()
|
||||
_EMPTY_NEW_CONNECTIONS: tuple[ControlConnection, ...] = ()
|
||||
|
||||
|
||||
def _tcp_addr(host: str, port: int | str) -> str:
|
||||
return f"tcp://{host}:{port}"
|
||||
|
||||
|
||||
def _apply_heartbeat(sock: zmq.Socket) -> None:
|
||||
sock.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS)
|
||||
sock.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS)
|
||||
sock.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Sockets:
|
||||
dealer: zmq.Socket
|
||||
monitor: zmq.Socket
|
||||
|
||||
|
||||
class ZmqConnection(ControlConnection):
|
||||
"""Bidirectional message channel to a single remote peer."""
|
||||
|
||||
def __init__(self, peer_id: str, sockets: _Sockets) -> None:
|
||||
super().__init__(peer_id)
|
||||
self._sockets = sockets
|
||||
self._closed = False
|
||||
self._inbox: list[dict] = []
|
||||
|
||||
def send(self, msg: dict) -> None:
|
||||
"""Send a msgpack-encoded message to this peer."""
|
||||
if self._closed:
|
||||
raise RuntimeError(
|
||||
f"ZmqConnection: send on closed connection to {self.peer_id}"
|
||||
)
|
||||
data = msgspec.msgpack.encode(msg)
|
||||
self._sockets.dealer.send(data)
|
||||
|
||||
def recv(self) -> Sequence[dict]:
|
||||
"""Drain and return all buffered incoming messages."""
|
||||
if not self._inbox:
|
||||
return _EMPTY_INBOX
|
||||
msgs = self._inbox
|
||||
self._inbox = []
|
||||
return msgs
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
return not self._closed
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
logger.info("ZmqConnection: closing connection to %s", self.peer_id)
|
||||
self._sockets.monitor.close()
|
||||
self._sockets.dealer.close()
|
||||
|
||||
def enqueue(self, msg: dict) -> None:
|
||||
"""Buffer an incoming message."""
|
||||
self._inbox.append(msg)
|
||||
|
||||
def mark_dead(self) -> None:
|
||||
"""Mark connection as disconnected."""
|
||||
self._closed = True
|
||||
|
||||
@property
|
||||
def monitor_socket(self) -> zmq.Socket:
|
||||
"""Monitor socket for disconnect detection (used by ZmqTransport)."""
|
||||
return self._sockets.monitor
|
||||
|
||||
|
||||
class ZmqTransport(ControlTransport):
|
||||
"""ZMQ implementation of ControlTransport.
|
||||
|
||||
Manages a ROUTER socket for accepting connections and DEALER sockets
|
||||
for outbound connections. Message-content agnostic.
|
||||
"""
|
||||
|
||||
def __init__(self, local_id: str, host: str, port: int) -> None:
|
||||
self._local_id = local_id
|
||||
self._closed = False
|
||||
|
||||
self._connections: dict[str, ZmqConnection] = {}
|
||||
self._pending_inbound: list[tuple[str, dict]] = []
|
||||
|
||||
self._zmq_ctx = zmq.Context()
|
||||
self._router: zmq.Socket = self._zmq_ctx.socket(zmq.ROUTER)
|
||||
_apply_heartbeat(self._router)
|
||||
bind_addr = _tcp_addr(host, port)
|
||||
self._router.bind(bind_addr)
|
||||
logger.info("ZmqTransport %s: ROUTER bound on %s", self._local_id, bind_addr)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# ZmqConnection lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def connect(self, peer_id: str) -> ZmqConnection:
|
||||
"""Open an outbound connection to a remote peer."""
|
||||
assert peer_id not in self._connections, (
|
||||
f"ZmqConnection to {peer_id} already exists"
|
||||
)
|
||||
logger.info(
|
||||
"ZmqTransport %s: opening OUTBOUND connection to %s",
|
||||
self._local_id,
|
||||
peer_id,
|
||||
)
|
||||
return self._open_connection(peer_id, direction="outbound")
|
||||
|
||||
def poll(self) -> Sequence[ControlConnection]:
|
||||
"""Process all pending I/O. Returns newly accepted connections.
|
||||
|
||||
- Receives messages (buffered in each connection's inbox)
|
||||
- Creates connections for new inbound peers (connect msg in inbox)
|
||||
- Checks monitors for disconnections
|
||||
- Removes and closes dead connections
|
||||
"""
|
||||
self._recv_router()
|
||||
self._check_monitors()
|
||||
|
||||
# Create connections for new inbound peers
|
||||
new_connections: list[ControlConnection] | None = None
|
||||
for sender_id, msg in self._pending_inbound:
|
||||
conn = self._connections.get(sender_id)
|
||||
if conn is None:
|
||||
logger.info(
|
||||
"ZmqTransport %s: accepting INBOUND connection from %s",
|
||||
self._local_id,
|
||||
sender_id,
|
||||
)
|
||||
conn = self._open_connection(sender_id, direction="inbound")
|
||||
if new_connections is None:
|
||||
new_connections = []
|
||||
new_connections.append(conn)
|
||||
conn.enqueue(msg)
|
||||
self._pending_inbound.clear()
|
||||
|
||||
# Remove dead connections
|
||||
for pid in [p for p, c in self._connections.items() if not c.alive]:
|
||||
self._connections.pop(pid).close()
|
||||
|
||||
return (
|
||||
new_connections if new_connections is not None else _EMPTY_NEW_CONNECTIONS
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
|
||||
for conn in self._connections.values():
|
||||
conn.close()
|
||||
self._connections.clear()
|
||||
|
||||
self._router.setsockopt(zmq.LINGER, 0)
|
||||
self._router.close()
|
||||
self._zmq_ctx.destroy(linger=0)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _open_connection(
|
||||
self, peer_id: str, direction: str = "outbound"
|
||||
) -> ZmqConnection:
|
||||
"""Create a DEALER socket + monitor and register the connection."""
|
||||
host, port_str = peer_id.rsplit(":", 1)
|
||||
dealer_addr = _tcp_addr(host, port_str)
|
||||
|
||||
logger.debug(
|
||||
"ZmqTransport %s: creating DEALER for %s peer %s -> %s",
|
||||
self._local_id,
|
||||
direction,
|
||||
peer_id,
|
||||
dealer_addr,
|
||||
)
|
||||
|
||||
dealer = self._zmq_ctx.socket(zmq.DEALER)
|
||||
_apply_heartbeat(dealer)
|
||||
dealer.identity = self._local_id.encode()
|
||||
|
||||
safe_id = peer_id.replace(":", "-").replace("/", "-")
|
||||
monitor_addr = f"inproc://p2p-monitor-{safe_id}"
|
||||
dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED)
|
||||
|
||||
monitor_sock = self._zmq_ctx.socket(zmq.PAIR)
|
||||
monitor_sock.connect(monitor_addr)
|
||||
|
||||
dealer.connect(dealer_addr)
|
||||
|
||||
sockets = _Sockets(dealer=dealer, monitor=monitor_sock)
|
||||
conn = ZmqConnection(peer_id, sockets)
|
||||
self._connections[peer_id] = conn
|
||||
logger.info(
|
||||
"ZmqTransport %s: %s connection established to %s (active connections: %d)",
|
||||
self._local_id,
|
||||
direction,
|
||||
peer_id,
|
||||
len(self._connections),
|
||||
)
|
||||
return conn
|
||||
|
||||
def _recv_router(self) -> None:
|
||||
"""Non-blocking: receive all pending messages from ROUTER."""
|
||||
while True:
|
||||
try:
|
||||
frames = self._router.recv_multipart(zmq.NOBLOCK)
|
||||
except zmq.Again:
|
||||
break
|
||||
except zmq.ZMQError as exc:
|
||||
logger.warning("ZmqTransport %s: recv error: %s", self._local_id, exc)
|
||||
break
|
||||
|
||||
if len(frames) != 2:
|
||||
logger.warning(
|
||||
"ZmqTransport %s: dropping message with %d frames (expected 2)",
|
||||
self._local_id,
|
||||
len(frames),
|
||||
)
|
||||
continue
|
||||
|
||||
identity, data = frames
|
||||
sender_id = identity.decode()
|
||||
|
||||
logger.debug(
|
||||
"ZmqTransport %s: ROUTER recv from %s (%d bytes)",
|
||||
self._local_id,
|
||||
sender_id,
|
||||
len(data),
|
||||
)
|
||||
|
||||
try:
|
||||
msg = msgspec.msgpack.decode(data)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"ZmqTransport %s: failed to decode message from %s: %s",
|
||||
self._local_id,
|
||||
sender_id,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
|
||||
conn = self._connections.get(sender_id)
|
||||
|
||||
if conn is not None:
|
||||
conn.enqueue(msg)
|
||||
else:
|
||||
self._pending_inbound.append((sender_id, msg))
|
||||
|
||||
def _check_monitors(self) -> None:
|
||||
"""Non-blocking: check all monitor sockets for disconnection."""
|
||||
for conn in self._connections.values():
|
||||
if not conn.alive:
|
||||
continue
|
||||
try:
|
||||
event = zmq.utils.monitor.recv_monitor_message(
|
||||
conn.monitor_socket, zmq.NOBLOCK
|
||||
)
|
||||
except zmq.Again:
|
||||
continue
|
||||
except zmq.ZMQError as exc:
|
||||
logger.warning(
|
||||
"ZmqTransport %s: monitor error for peer %s: %s",
|
||||
self._local_id,
|
||||
conn.peer_id,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
|
||||
if event["event"] == zmq.EVENT_DISCONNECTED:
|
||||
logger.debug(
|
||||
"ZmqTransport %s: peer %s disconnected",
|
||||
self._local_id,
|
||||
conn.peer_id,
|
||||
)
|
||||
conn.mark_dead()
|
||||
@@ -0,0 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.base import DataTransport, PollResult
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
|
||||
|
||||
__all__ = [
|
||||
"DataTransport",
|
||||
"NixlTransport",
|
||||
"PollResult",
|
||||
]
|
||||
@@ -0,0 +1,259 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base class for the P2P data-plane transport.
|
||||
|
||||
The data plane handles RDMA (or similar) block transfers between peers.
|
||||
It is independent of the control plane — the control plane establishes
|
||||
who can talk to whom, then the data plane moves blocks at wire speed.
|
||||
|
||||
Architecture
|
||||
------------
|
||||
|
||||
DataTransport (one per node)
|
||||
├── owns the local KV block memory region
|
||||
├── registers remote peers (add_remote_peer / remove_remote_peer)
|
||||
├── submits block writes (write_blocks → transfer_id)
|
||||
├── polls for completion (poll → done/failed IDs)
|
||||
└── cancels inflight transfers (cancel)
|
||||
|
||||
Memory model
|
||||
------------
|
||||
|
||||
The local node exposes a contiguous block region:
|
||||
|
||||
base_addr ──► ┌─────────────┐ block 0
|
||||
├─────────────┤ block 1
|
||||
├─────────────┤ ...
|
||||
└─────────────┘ block (num_blocks - 1)
|
||||
|
||||
Each block is block_len bytes.
|
||||
|
||||
Remote peers expose the same layout. write_blocks() copies local
|
||||
blocks to a remote peer's block region by index:
|
||||
|
||||
write_blocks("peer:1", local_idxs=[0, 3], remote_idxs=[5, 7])
|
||||
→ writes local block 0 → remote block 5
|
||||
local block 3 → remote block 7
|
||||
|
||||
Transfer lifecycle
|
||||
------------------
|
||||
|
||||
1. Register remote peer: add_remote_peer(peer_id, metadata, ...)
|
||||
- Provides the remote memory layout so transfers can target it
|
||||
2. Submit: write_blocks(peer_id, local_idxs, remote_idxs) → int
|
||||
- Returns a transfer_id (opaque int) for tracking
|
||||
- Returns None if peer not registered or submission fails
|
||||
3. Poll: poll() → PollResult(done=[...], failed=[...])
|
||||
- Returns transfer_ids that completed or failed since last poll
|
||||
- Completed transfers are automatically cleaned up
|
||||
4. Cancel: cancel(transfer_ids, mode="immediate" | "wait")
|
||||
- Best-effort cancellation of inflight transfers
|
||||
- mode="wait" returns ids still in PROC/PEND so the caller can poll
|
||||
them to completion
|
||||
|
||||
Implementor contracts
|
||||
---------------------
|
||||
|
||||
- write_blocks() must not block. The transfer runs asynchronously.
|
||||
- poll() must be called periodically. It drives completion checking.
|
||||
- transfer_ids are unique across the lifetime of the transport.
|
||||
- add_remote_peer() must be called before write_blocks() to that peer.
|
||||
- get_agent_metadata() returns opaque bytes that the remote peer
|
||||
needs to call add_remote_peer() (e.g., RDMA connection info).
|
||||
- config_fingerprint is a content hash of the model configuration.
|
||||
Peers with different fingerprints are incompatible and must not
|
||||
exchange blocks (validated during the control-plane handshake).
|
||||
- close() releases all resources (memory registrations, handles).
|
||||
After close(), no other methods may be called.
|
||||
|
||||
Threading model: no background threads. All I/O driven by poll().
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import hashlib
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import Literal, NamedTuple
|
||||
|
||||
CancelMode = Literal["immediate", "wait"]
|
||||
|
||||
|
||||
class PollResult(NamedTuple):
|
||||
"""Result of polling inflight transfers.
|
||||
|
||||
Attributes:
|
||||
done: Transfer IDs that completed successfully.
|
||||
failed: Transfer IDs that failed (error, timeout, etc.).
|
||||
"""
|
||||
|
||||
done: Sequence[int]
|
||||
failed: Sequence[int]
|
||||
|
||||
|
||||
class DataTransport(ABC):
|
||||
"""Abstract data-plane transport for RDMA-style block transfers.
|
||||
|
||||
Owns the local KV block memory region and manages transfers to/from
|
||||
registered remote peers.
|
||||
|
||||
Construction:
|
||||
view: A 2D memoryview (num_blocks × block_len bytes) over the
|
||||
local KV cache block storage.
|
||||
config_fields: Dict of model config values used to compute the
|
||||
compatibility fingerprint. None → empty fingerprint
|
||||
(compatible with any peer).
|
||||
"""
|
||||
|
||||
def __init__(self, view: memoryview, config_fields: dict | None = None) -> None:
|
||||
assert view.shape is not None
|
||||
self._view = view
|
||||
self._base_addr = ctypes.addressof(ctypes.c_char.from_buffer(view))
|
||||
self._num_blocks = view.shape[0]
|
||||
self._block_len = view.shape[1]
|
||||
self._config_fingerprint = self._compute_fingerprint(config_fields)
|
||||
|
||||
@property
|
||||
def base_addr(self) -> int:
|
||||
"""Base address of the local block memory region."""
|
||||
return self._base_addr
|
||||
|
||||
@property
|
||||
def num_blocks(self) -> int:
|
||||
"""Number of blocks in the local region."""
|
||||
return self._num_blocks
|
||||
|
||||
@property
|
||||
def block_len(self) -> int:
|
||||
"""Size of each block in bytes."""
|
||||
return self._block_len
|
||||
|
||||
@property
|
||||
def config_fingerprint(self) -> str:
|
||||
"""Content-hash of the model configuration (hex string).
|
||||
|
||||
Peers must have matching fingerprints to exchange blocks.
|
||||
Empty string means no fingerprint (always compatible).
|
||||
"""
|
||||
return self._config_fingerprint
|
||||
|
||||
@staticmethod
|
||||
def _compute_fingerprint(config_fields: dict | None) -> str:
|
||||
if not config_fields:
|
||||
return ""
|
||||
canonical = json.dumps(config_fields, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode()).hexdigest()[:16]
|
||||
|
||||
@abstractmethod
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
"""Return opaque metadata needed by remote peers to connect.
|
||||
|
||||
The returned bytes are sent during the control-plane handshake
|
||||
and passed to the remote peer's add_remote_peer().
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def add_remote_peer(
|
||||
self,
|
||||
peer_id: str,
|
||||
agent_metadata: bytes,
|
||||
base_addr: int,
|
||||
num_blocks: int,
|
||||
block_len: int,
|
||||
) -> None:
|
||||
"""Register a remote peer for block transfers.
|
||||
|
||||
Must be called before write_blocks() to this peer.
|
||||
|
||||
Args:
|
||||
peer_id: Unique identifier for the remote peer.
|
||||
agent_metadata: Opaque bytes from the peer's get_agent_metadata().
|
||||
base_addr: Base address of the peer's block memory region.
|
||||
num_blocks: Number of blocks in the peer's region.
|
||||
block_len: Size of each block (must match local block_len).
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def remove_remote_peer(self, peer_id: str) -> None:
|
||||
"""Unregister a remote peer and release associated resources.
|
||||
|
||||
Inflight transfers to this peer should be cancelled first.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def write_blocks(
|
||||
self,
|
||||
peer_id: str,
|
||||
local_idxs: list[int],
|
||||
remote_idxs: list[int],
|
||||
) -> int | None:
|
||||
"""Submit a WRITE transfer: local blocks → remote peer's blocks.
|
||||
|
||||
Args:
|
||||
peer_id: Target peer (must be registered via add_remote_peer).
|
||||
local_idxs: Indexes of local blocks to read from.
|
||||
remote_idxs: Indexes of remote blocks to write to.
|
||||
Must be same length as local_idxs.
|
||||
|
||||
Returns:
|
||||
A unique transfer_id (int) to track this transfer, or
|
||||
None if the peer is not registered or submission failed.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def poll(self) -> PollResult:
|
||||
"""Poll all inflight transfers for completion.
|
||||
|
||||
Returns:
|
||||
PollResult with lists of completed and failed transfer_ids.
|
||||
Completed/failed transfers are removed from the inflight set.
|
||||
|
||||
Must be called periodically to drive progress checking.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def cancel(
|
||||
self,
|
||||
transfer_ids: Iterable[int],
|
||||
mode: CancelMode = "immediate",
|
||||
) -> list[int]:
|
||||
"""Cancel inflight transfers by their IDs.
|
||||
|
||||
Best-effort: transfers that already completed are ignored.
|
||||
|
||||
Args:
|
||||
transfer_ids: IDs to cancel. Unknown IDs are ignored.
|
||||
mode:
|
||||
"immediate" (default): pop and release each handle and
|
||||
return []. Matches the legacy fire-and-forget
|
||||
behavior — the caller does not wait for the
|
||||
underlying transfer to drain.
|
||||
"wait": attempt to release each handle. If the release
|
||||
cannot complete because the transfer is still
|
||||
PROC/PEND, the entry stays in the inflight set and
|
||||
its id is included in the returned list. The
|
||||
caller is expected to keep calling poll() until
|
||||
every returned id surfaces in done/failed.
|
||||
|
||||
Returns:
|
||||
For mode="wait", the subset of *transfer_ids* still
|
||||
tracked as inflight after the cancel attempt. For
|
||||
mode="immediate", always [].
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Release all resources (registrations, handles, memory).
|
||||
|
||||
Cancels any remaining inflight transfers. Idempotent.
|
||||
After close(), no other methods may be called.
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
NixlTransport: Data-plane transport for RDMA-based KV block transfers via NIXL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
from vllm.distributed.nixl_utils import NixlWrapper as _NixlAgent
|
||||
from vllm.distributed.nixl_utils import nixl_agent_config as _NixlAgentConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.base import (
|
||||
CancelMode,
|
||||
DataTransport,
|
||||
PollResult,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Shared sentinel returned by poll() in the steady state (no inflight, or
|
||||
# no transfer changed state since the last poll). Tuples make it immutable;
|
||||
# callers only iterate / membership-test / equality-check.
|
||||
_EMPTY_POLL_RESULT: PollResult = PollResult(done=(), failed=())
|
||||
|
||||
|
||||
class NixlTransport(DataTransport):
|
||||
"""Manages a NIXL agent, memory registration, and block transfers.
|
||||
|
||||
Wraps the NIXL C library behind a Python interface so the rest of the
|
||||
P2P tier code never touches NIXL types directly. Tracks inflight
|
||||
handles internally and returns completed/failed tags on poll.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
local_id: str,
|
||||
view: memoryview,
|
||||
config_fields: dict | None = None,
|
||||
backends: list[str] | None = None,
|
||||
num_threads: int = 4,
|
||||
) -> None:
|
||||
super().__init__(view, config_fields=config_fields)
|
||||
self._local_id = local_id
|
||||
self._backends = list(backends) if backends else ["UCX"]
|
||||
self._num_threads = num_threads
|
||||
self._agent: Any = None
|
||||
self._reg: Any = None
|
||||
self._local_dlist: Any = None
|
||||
self._remote_dlists: dict[str, object] = {}
|
||||
self._peer_nixl_names: dict[str, str] = {}
|
||||
self._inflight: dict[int, object] = {} # transfer_id → handle
|
||||
self._next_id = itertools.count()
|
||||
|
||||
self._init(view)
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._agent is not None
|
||||
|
||||
def _init(self, view: memoryview) -> None:
|
||||
if _NixlAgent is None:
|
||||
return
|
||||
|
||||
non_ucx_backends = [b for b in self._backends if b != "UCX"]
|
||||
if non_ucx_backends:
|
||||
cfg = _NixlAgentConfig(backends=self._backends, capture_telemetry=True)
|
||||
logger.info(
|
||||
"NixlTransport %s: NIXL backends=%s",
|
||||
self._local_id,
|
||||
self._backends,
|
||||
)
|
||||
else:
|
||||
cfg = _NixlAgentConfig(
|
||||
num_threads=self._num_threads, capture_telemetry=True
|
||||
)
|
||||
logger.info(
|
||||
"NixlTransport %s: NIXL backends=[UCX] num_threads=%d",
|
||||
self._local_id,
|
||||
self._num_threads,
|
||||
)
|
||||
self._agent = _NixlAgent(self._local_id, cfg)
|
||||
|
||||
total_size = self._num_blocks * self._block_len
|
||||
reg_descs = [(self._base_addr, total_size, 0, "")]
|
||||
self._reg = self._agent.register_memory(reg_descs, mem_type="DRAM")
|
||||
|
||||
block_tuples = [
|
||||
(self._base_addr + i * self._block_len, self._block_len, 0)
|
||||
for i in range(self._num_blocks)
|
||||
]
|
||||
xfer_dlist = self._agent.get_xfer_descs(block_tuples, mem_type="DRAM")
|
||||
self._local_dlist = self._agent.prep_xfer_dlist("NIXL_INIT_AGENT", xfer_dlist)
|
||||
logger.info(
|
||||
"NixlTransport %s: registered %d blocks", self._local_id, self._num_blocks
|
||||
)
|
||||
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
assert self._agent is not None
|
||||
return self._agent.get_agent_metadata()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Peer management
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_remote_peer(
|
||||
self,
|
||||
peer_id: str,
|
||||
agent_metadata: bytes,
|
||||
base_addr: int,
|
||||
num_blocks: int,
|
||||
block_len: int,
|
||||
) -> None:
|
||||
nixl_name = self._agent.add_remote_agent(agent_metadata)
|
||||
block_descs = [
|
||||
(base_addr + i * block_len, block_len, 0) for i in range(num_blocks)
|
||||
]
|
||||
xfer_dlist = self._agent.get_xfer_descs(block_descs, mem_type="DRAM")
|
||||
remote_dlist = self._agent.prep_xfer_dlist(nixl_name, xfer_dlist)
|
||||
self._peer_nixl_names[peer_id] = nixl_name
|
||||
self._remote_dlists[peer_id] = remote_dlist
|
||||
|
||||
def remove_remote_peer(self, peer_id: str) -> None:
|
||||
nixl_name = self._peer_nixl_names.pop(peer_id, None)
|
||||
dlist = self._remote_dlists.pop(peer_id, None)
|
||||
if self._agent is not None:
|
||||
if dlist is not None:
|
||||
self._agent.release_dlist_handle(dlist)
|
||||
if nixl_name:
|
||||
self._agent.remove_remote_agent(nixl_name)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Transfer submission and polling
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def write_blocks(
|
||||
self,
|
||||
peer_id: str,
|
||||
local_idxs: list[int],
|
||||
remote_idxs: list[int],
|
||||
) -> int | None:
|
||||
"""Submit a WRITE transfer to *peer_id*.
|
||||
|
||||
Returns a transfer ID, or None if the peer is not registered.
|
||||
The ID is returned via poll() when the transfer completes or fails.
|
||||
"""
|
||||
remote_dlist = self._remote_dlists.get(peer_id)
|
||||
if remote_dlist is None:
|
||||
logger.warning(
|
||||
"NixlTransport %s: write_blocks NO REMOTE DLIST for peer=%s "
|
||||
"(known peers=%s)",
|
||||
self._local_id,
|
||||
peer_id,
|
||||
list(self._remote_dlists.keys()),
|
||||
)
|
||||
return None
|
||||
logger.debug(
|
||||
"NixlTransport %s: write_blocks NIXL.transfer peer=%s blocks=%d",
|
||||
self._local_id,
|
||||
peer_id,
|
||||
len(local_idxs),
|
||||
)
|
||||
handle = self._agent.make_prepped_xfer(
|
||||
"WRITE",
|
||||
self._local_dlist,
|
||||
local_idxs,
|
||||
remote_dlist,
|
||||
remote_idxs,
|
||||
)
|
||||
self._agent.transfer(handle)
|
||||
transfer_id = next(self._next_id)
|
||||
self._inflight[transfer_id] = handle
|
||||
return transfer_id
|
||||
|
||||
def poll(self) -> PollResult:
|
||||
"""Poll all inflight transfers.
|
||||
|
||||
Returns PollResult(done=..., failed=...) with transfer IDs.
|
||||
Completed handles are released automatically.
|
||||
"""
|
||||
if not self._inflight:
|
||||
return _EMPTY_POLL_RESULT
|
||||
|
||||
done_ids: list[int] | None = None
|
||||
failed_ids: list[int] | None = None
|
||||
|
||||
for transfer_id, handle in self._inflight.items():
|
||||
try:
|
||||
state = self._agent.check_xfer_state(handle)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"NixlTransport %s: check_xfer_state failed for transfer_id=%d: %s",
|
||||
self._local_id,
|
||||
transfer_id,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
if state == "DONE":
|
||||
if done_ids is None:
|
||||
done_ids = []
|
||||
done_ids.append(transfer_id)
|
||||
elif state not in ("PROC", "PEND"):
|
||||
if failed_ids is None:
|
||||
failed_ids = []
|
||||
failed_ids.append(transfer_id)
|
||||
|
||||
if done_ids is None and failed_ids is None:
|
||||
return _EMPTY_POLL_RESULT
|
||||
|
||||
handles_to_release = []
|
||||
for tid in done_ids or ():
|
||||
handles_to_release.append(self._inflight.pop(tid))
|
||||
for tid in failed_ids or ():
|
||||
handles_to_release.append(self._inflight.pop(tid))
|
||||
self._release_handles(handles_to_release)
|
||||
|
||||
return PollResult(
|
||||
done=done_ids if done_ids is not None else _EMPTY_POLL_RESULT.done,
|
||||
failed=failed_ids if failed_ids is not None else _EMPTY_POLL_RESULT.failed,
|
||||
)
|
||||
|
||||
def cancel(
|
||||
self,
|
||||
transfer_ids: Iterable[int],
|
||||
mode: CancelMode = "immediate",
|
||||
) -> list[int]:
|
||||
"""Cancel inflight transfers by their IDs.
|
||||
|
||||
See ``DataTransport.cancel`` for the contract. In "wait" mode,
|
||||
transfers whose ``release_xfer_handle`` raises (NIXL could not
|
||||
complete the abort because the backend is still draining) stay
|
||||
in ``self._inflight`` so a later ``poll()`` will observe them.
|
||||
"""
|
||||
if mode == "immediate":
|
||||
handles = [
|
||||
self._inflight.pop(tid) for tid in transfer_ids if tid in self._inflight
|
||||
]
|
||||
self._release_handles(handles)
|
||||
return []
|
||||
|
||||
still_inflight: list[int] = []
|
||||
for tid in transfer_ids:
|
||||
handle = self._inflight.get(tid)
|
||||
if handle is None:
|
||||
continue
|
||||
try:
|
||||
self._agent.release_xfer_handle(handle)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"NixlTransport %s: cancel pending for transfer_id=%d: %s",
|
||||
self._local_id,
|
||||
tid,
|
||||
exc,
|
||||
)
|
||||
still_inflight.append(tid)
|
||||
continue
|
||||
del self._inflight[tid]
|
||||
return still_inflight
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def close(self) -> None:
|
||||
if self._agent is None:
|
||||
return
|
||||
self._release_handles(list(self._inflight.values()))
|
||||
self._inflight.clear()
|
||||
for peer_id in list(self._remote_dlists):
|
||||
self.remove_remote_peer(peer_id)
|
||||
if self._local_dlist is not None:
|
||||
self._agent.release_dlist_handle(self._local_dlist)
|
||||
self._local_dlist = None
|
||||
if self._reg is not None:
|
||||
self._agent.deregister_memory(self._reg)
|
||||
self._reg = None
|
||||
self._agent = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _release_handles(self, handles: list[object]) -> None:
|
||||
if self._agent is None:
|
||||
return
|
||||
for handle in handles:
|
||||
try:
|
||||
self._agent.release_xfer_handle(handle)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"NixlTransport %s: release_xfer_handle failed: %s",
|
||||
self._local_id,
|
||||
exc,
|
||||
)
|
||||
@@ -0,0 +1,664 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
P2PSecondaryTierManager: Secondary tier for P2P KV cache sharing.
|
||||
|
||||
Owns transports and a single bidirectional P2PSession per remote peer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobMetadata,
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport
|
||||
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport, NixlTransport
|
||||
from vllm.v1.kv_offload.tiering.p2p.session import P2PSession
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.base import ControlConnection
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Reap unbound store batches that have been parked without a FetchMsg
|
||||
# binding them to a session for longer than this. Protects against the
|
||||
# prefiller buffering blocks for a decoder that never asks (decoder died,
|
||||
# network partition, lost kv_request_id). Must be longer than the per-store
|
||||
# deadline so the store-timeout path fires first for individual jobs.
|
||||
_UNBOUND_STORE_TIMEOUT_S = 60.0
|
||||
|
||||
# Time we wait during shutdown for inflight transfers to drain via
|
||||
# cancel(mode="wait") before falling back to mode="immediate". Bounded
|
||||
# so a wedged peer can't hang shutdown.
|
||||
_SHUTDOWN_DRAIN_TIMEOUT_S = 3.0
|
||||
|
||||
# Sleep between iterations of the bounded drain loops in drain_jobs() and
|
||||
# _drain_inflight_for_shutdown(). Short enough to keep latency low, long
|
||||
# enough to avoid busy-spinning the scheduler thread.
|
||||
_DRAIN_SLEEP_S = 0.001
|
||||
|
||||
|
||||
def _prefill_params(kv_params: dict | None) -> dict | None:
|
||||
"""Return the ``prefill`` sub-dict, or None if absent.
|
||||
|
||||
Set on decoder requests; carries kv_request_id, remote_host, remote_port.
|
||||
"""
|
||||
if not kv_params:
|
||||
return None
|
||||
return kv_params.get("prefill")
|
||||
|
||||
|
||||
def _decode_params(kv_params: dict | None) -> dict | None:
|
||||
"""Return the ``decode`` sub-dict, or None if absent.
|
||||
|
||||
Set on prefiller requests; carries kv_request_id.
|
||||
"""
|
||||
if not kv_params:
|
||||
return None
|
||||
return kv_params.get("decode")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _UnboundStoreBatch:
|
||||
"""A submit_store batch parked at the manager before any peer has fetched.
|
||||
|
||||
Indexed by kv_request_id only — the prefiller no longer learns the peer
|
||||
identity at store time. When a FetchMsg(kv_request_id) arrives on some
|
||||
session, the manager binds the kv_request_id to that session and replays
|
||||
every parked batch into ServerRole via session.add_stored_blocks.
|
||||
"""
|
||||
|
||||
job_id: int
|
||||
keys: list[OffloadKey]
|
||||
block_ids: Sequence[int]
|
||||
submitted_at: float = field(default_factory=time.monotonic)
|
||||
|
||||
|
||||
class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
"""Secondary tier for P2P KV cache sharing.
|
||||
|
||||
A single P2PSession per remote peer handles both client-role (loading
|
||||
blocks from the peer) and server-role (serving blocks to the peer)
|
||||
over the same control connection.
|
||||
|
||||
Single-threaded: every public method runs on the scheduler thread, and
|
||||
the engine drives polling via ``get_finished_jobs()`` once per step.
|
||||
``has_pending_work()`` keeps the engine ticking so the control transport
|
||||
and existing sessions are polled even when no requests are scheduled.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
offloading_spec: OffloadingSpec,
|
||||
primary_kv_view: memoryview,
|
||||
tier_type: str = "p2p",
|
||||
host: str = "0.0.0.0",
|
||||
port: int = 7777,
|
||||
backends: list[str] | None = None,
|
||||
num_threads: int = 4,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""Initialize the P2P secondary tier manager.
|
||||
|
||||
All keyword arguments after ``primary_kv_view`` come from the
|
||||
``secondary_tiers`` entry in ``kv_connector_extra_config``. See
|
||||
``docs/features/kv_offloading_usage.md`` for the user-facing
|
||||
configuration reference.
|
||||
|
||||
Args:
|
||||
offloading_spec: Owning ``OffloadingSpec`` (provides
|
||||
``vllm_config`` and the offloaded block layout).
|
||||
primary_kv_view: Memoryview over the CPU primary tier; the
|
||||
NIXL agent registers this region for RDMA transfers.
|
||||
tier_type: Tier identifier (defaults to ``"p2p"``).
|
||||
host: Address the ZMQ control socket binds to.
|
||||
port: Port for the ZMQ control socket. Must be reachable
|
||||
from peers.
|
||||
backends: NIXL transport backends (e.g. ``["UCX"]``,
|
||||
``["MOONCAKE"]``, ``["LIBFABRIC"]``). Defaults to
|
||||
``["UCX"]``. When any non-UCX backend is requested, the
|
||||
NIXL agent is initialized with ``backends=...``;
|
||||
otherwise it falls back to a UCX-only agent with
|
||||
``num_threads`` threads.
|
||||
num_threads: NIXL agent worker threads for the UCX-only
|
||||
branch. Ignored when ``backends`` contains a non-UCX
|
||||
entry.
|
||||
**kwargs: Reserved for future tier-specific options.
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
port = int(port)
|
||||
self._local_id = f"{host}:{port}"
|
||||
|
||||
config_fields = FileMapper.from_offloading_spec(
|
||||
root_dir="",
|
||||
offloading_spec=offloading_spec,
|
||||
gpu_blocks_per_file=offloading_spec.block_size_factor,
|
||||
parallel_agnostic=True,
|
||||
).get_run_config()
|
||||
self._data: DataTransport = NixlTransport(
|
||||
self._local_id,
|
||||
primary_kv_view,
|
||||
config_fields=config_fields,
|
||||
backends=backends,
|
||||
num_threads=int(num_threads),
|
||||
)
|
||||
self._control: ControlTransport = ZmqTransport(self._local_id, host, port)
|
||||
|
||||
self._sessions: dict[str, P2PSession] = {}
|
||||
# kv_request_id → session, set when the bound session has received
|
||||
# FetchMsg for that id. submit_store after binding routes directly
|
||||
# to the session; before binding, batches are parked in
|
||||
# _unbound_stores below. Stays in sync with _sessions: entries
|
||||
# pointing to a reaped session are purged in _reap_dead_sessions.
|
||||
self._kv_to_session: dict[str, P2PSession] = {}
|
||||
# kv_request_id → list of batches submit_store'd before any peer
|
||||
# asked for that id. Drained into a session by _on_session_fetch
|
||||
# when the corresponding FetchMsg arrives, or surfaced as failures
|
||||
# by _reap_unbound_stores after _UNBOUND_STORE_TIMEOUT_S.
|
||||
self._unbound_stores: dict[str, list[_UnboundStoreBatch]] = {}
|
||||
|
||||
self._finished_jobs: list[JobResult] = []
|
||||
# kv_request_ids that hit a transport/session failure; On load lookup()
|
||||
# rejects them so the request falls back to local prefill.
|
||||
self._failed_req_ids: set[str] = set()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# SecondaryTierManager interface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
prefill = _prefill_params(req_context.kv_transfer_params)
|
||||
if (
|
||||
not prefill
|
||||
or not prefill.get("remote_host")
|
||||
or not prefill.get("remote_port")
|
||||
or not prefill.get("kv_request_id")
|
||||
):
|
||||
return LookupResult.MISS
|
||||
|
||||
kv_request_id = prefill["kv_request_id"]
|
||||
if kv_request_id in self._failed_req_ids:
|
||||
return LookupResult.MISS
|
||||
return LookupResult.HIT
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
"""Open the outbound session toward the producer if needed.
|
||||
|
||||
On the decoder side (``prefill`` set), open a session toward the
|
||||
producer at remote_host:remote_port so submit_load can issue
|
||||
FetchMsg as soon as it fires. On the prefiller side, sessions
|
||||
are created when the consumer's inbound connection arrives in
|
||||
_accept_new_peers — submit_store no longer pre-creates anything.
|
||||
"""
|
||||
prefill = _prefill_params(req_context.kv_transfer_params)
|
||||
if prefill:
|
||||
peer_id = self._remote_id_from_params(prefill)
|
||||
if peer_id:
|
||||
self._get_or_create_session(peer_id)
|
||||
return RequestOffloadingContext()
|
||||
|
||||
@override
|
||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||
"""Cancels pending loads and prunes session-scoped state.
|
||||
|
||||
Decoder side (``prefill`` set): looks up the session by peer_id
|
||||
because the producer's address is what addresses the client-role
|
||||
load to cancel. Prefiller side (``decode`` set): looks up via
|
||||
kv_request_id because peer_id is no longer carried on store-time
|
||||
kv_transfer_params; if a session has bound the id, finish it. If
|
||||
no session has bound the id yet, this is a no-op: parked batches
|
||||
in `_unbound_stores` are left in place and cleaned up only by
|
||||
`_reap_unbound_stores` after `_UNBOUND_STORE_TIMEOUT_S`.
|
||||
"""
|
||||
kv_params = req_context.kv_transfer_params
|
||||
if not kv_params:
|
||||
return
|
||||
prefill = _prefill_params(kv_params)
|
||||
decode = _decode_params(kv_params)
|
||||
kv_request_id = (prefill or decode or {}).get("kv_request_id")
|
||||
if not kv_request_id:
|
||||
return
|
||||
self._failed_req_ids.discard(kv_request_id)
|
||||
|
||||
if prefill:
|
||||
peer_id = self._remote_id_from_params(prefill)
|
||||
if peer_id:
|
||||
session = self._sessions.get(peer_id)
|
||||
if session is not None:
|
||||
session.finish_request(kv_request_id)
|
||||
return
|
||||
|
||||
# Prefiller-side finish: identify the session via kv_request_id.
|
||||
session = self._kv_to_session.pop(kv_request_id, None)
|
||||
if session is not None:
|
||||
session.finish_request(kv_request_id)
|
||||
return
|
||||
|
||||
@override
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
job_id = job_metadata.job_id
|
||||
keys = list(job_metadata.keys)
|
||||
block_ids = job_metadata.block_ids
|
||||
|
||||
assert len(keys) == len(block_ids)
|
||||
|
||||
kv_params = job_metadata.req_context.kv_transfer_params
|
||||
decode = _decode_params(kv_params)
|
||||
logger.debug(
|
||||
"P2P %s: submit_store ENTRY job_id=%d blocks=%d decode=%s kv_request_id=%s",
|
||||
self._local_id,
|
||||
job_id,
|
||||
len(block_ids),
|
||||
decode is not None,
|
||||
(decode or {}).get("kv_request_id"),
|
||||
)
|
||||
# Absent ``decode`` block => not a remote-decode request: succeed
|
||||
# locally without parking. An empty/malformed dict is still a
|
||||
# remote-decode signal and must fail the missing-id check below.
|
||||
if decode is None:
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=True))
|
||||
return
|
||||
|
||||
kv_request_id = decode.get("kv_request_id")
|
||||
if not kv_request_id:
|
||||
logger.warning(
|
||||
"P2P %s: submit_store missing kv_request_id",
|
||||
self._local_id,
|
||||
)
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
return
|
||||
|
||||
# Fast path: a session has already received FetchMsg for this id,
|
||||
# so we can route the batch straight into its ServerRole.
|
||||
session = self._kv_to_session.get(kv_request_id)
|
||||
if session is not None:
|
||||
session.add_stored_blocks(kv_request_id, keys, block_ids, job_id)
|
||||
return
|
||||
|
||||
# No session bound yet — park the batch keyed by kv_request_id.
|
||||
# _on_session_fetch drains it on the first FetchMsg; if no peer
|
||||
# ever asks, _reap_unbound_stores surfaces the job as failed.
|
||||
self._unbound_stores.setdefault(kv_request_id, []).append(
|
||||
_UnboundStoreBatch(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=block_ids,
|
||||
)
|
||||
)
|
||||
logger.debug(
|
||||
"P2P %s: parked submit_store kv_request_id=%s job_id=%d blocks=%d",
|
||||
self._local_id,
|
||||
kv_request_id,
|
||||
job_id,
|
||||
len(block_ids),
|
||||
)
|
||||
|
||||
@override
|
||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
||||
job_id = job_metadata.job_id
|
||||
keys = list(job_metadata.keys)
|
||||
block_ids = job_metadata.block_ids
|
||||
|
||||
prefill = _prefill_params(job_metadata.req_context.kv_transfer_params)
|
||||
logger.debug(
|
||||
"P2P %s: submit_load ENTRY job_id=%d blocks=%d kv_request_id=%s peer=%s",
|
||||
self._local_id,
|
||||
job_id,
|
||||
len(block_ids),
|
||||
(prefill or {}).get("kv_request_id"),
|
||||
self._remote_id_from_params(prefill or {}),
|
||||
)
|
||||
if (
|
||||
not prefill
|
||||
or not prefill.get("remote_host")
|
||||
or not prefill.get("remote_port")
|
||||
or not prefill.get("kv_request_id")
|
||||
):
|
||||
logger.debug(
|
||||
"P2P %s: submit_load job_id=%d FAILED missing prefill params",
|
||||
self._local_id,
|
||||
job_id,
|
||||
)
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
return
|
||||
|
||||
kv_request_id = prefill["kv_request_id"]
|
||||
peer_id = self._remote_id_from_params(prefill)
|
||||
assert peer_id is not None # guaranteed by prefill checks above
|
||||
|
||||
if not keys:
|
||||
logger.debug(
|
||||
"P2P %s: submit_load job_id=%d short-circuit success (no keys)",
|
||||
self._local_id,
|
||||
job_id,
|
||||
)
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=True))
|
||||
return
|
||||
|
||||
session = self._sessions.get(peer_id)
|
||||
if session is None:
|
||||
logger.warning(
|
||||
"P2P %s: submit_load job_id=%d NO SESSION for peer=%s",
|
||||
self._local_id,
|
||||
job_id,
|
||||
peer_id,
|
||||
)
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
self._failed_req_ids.add(kv_request_id)
|
||||
return
|
||||
logger.debug(
|
||||
"P2P %s: submit_load job_id=%d -> request_blocks peer=%s "
|
||||
"kv_request_id=%s blocks=%d session_ready=%s",
|
||||
self._local_id,
|
||||
job_id,
|
||||
peer_id,
|
||||
kv_request_id,
|
||||
len(block_ids),
|
||||
session.ready,
|
||||
)
|
||||
session.request_blocks(job_id, kv_request_id, keys, block_ids)
|
||||
|
||||
@override
|
||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||
# Drive one polling sweep on the scheduler thread, then hand off
|
||||
# whatever has accumulated. The engine calls this once per step
|
||||
# (and keeps stepping while has_pending_work() is True).
|
||||
self._poll_once()
|
||||
result = self._finished_jobs
|
||||
self._finished_jobs = []
|
||||
return result
|
||||
|
||||
@override
|
||||
def has_pending_work(self) -> bool:
|
||||
# The engine tick is the only driver of _control.poll() and
|
||||
# session.poll(); without it we miss new peer connects and
|
||||
# inbound fetch messages on existing sessions. Keep the engine
|
||||
# ticking for the lifetime of this manager.
|
||||
return True
|
||||
|
||||
@override
|
||||
def drain_jobs(self) -> None:
|
||||
"""Block until every submitted load/store job has completed or failed.
|
||||
|
||||
Loops calling ``_poll_once()`` until no session has outstanding
|
||||
inbound loads or in-flight outbound stores. Mid-flight transfers
|
||||
are NOT cancelled — the caller (``TieringOffloadingManager.reset_cache``)
|
||||
needs the primary memoryview to be quiescent, not aborted. Results
|
||||
accumulate in ``_finished_jobs`` and are surfaced by the next
|
||||
``get_finished_jobs()`` call.
|
||||
"""
|
||||
start = time.monotonic()
|
||||
warned = False
|
||||
while True:
|
||||
self._poll_once()
|
||||
pending = any(
|
||||
s._client._inbound or s._server._inflight
|
||||
for s in self._sessions.values()
|
||||
)
|
||||
if not pending:
|
||||
return
|
||||
if not warned and time.monotonic() - start > 5.0:
|
||||
logger.warning(
|
||||
"P2PSecondaryTierManager.drain_jobs: still draining "
|
||||
"after 5s; a stuck transfer will block the engine.",
|
||||
)
|
||||
warned = True
|
||||
time.sleep(_DRAIN_SLEEP_S)
|
||||
|
||||
@override
|
||||
def on_schedule_end(self) -> None:
|
||||
return
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _remote_id_from_params(role_params: dict) -> str | None:
|
||||
"""Build peer_id from a role-scoped sub-dict (``prefill``/``p2p``)."""
|
||||
host = role_params.get("remote_host")
|
||||
port = role_params.get("remote_port")
|
||||
if host and port:
|
||||
return f"{host}:{port}"
|
||||
return None
|
||||
|
||||
def _get_or_create_session(self, peer_id: str) -> P2PSession:
|
||||
"""Return the existing session for peer_id, or open one outbound.
|
||||
|
||||
Decoder-side helper for on_new_request: when ``prefill`` is set,
|
||||
the consumer must reach the producer at peer_id. If we already
|
||||
have a session toward that peer (from a prior load or a
|
||||
peer-initiated inbound), reuse it; otherwise open an outbound
|
||||
ControlConnection and build a connected session.
|
||||
"""
|
||||
session = self._sessions.get(peer_id)
|
||||
if session is not None:
|
||||
return session
|
||||
conn = self._control.connect(peer_id)
|
||||
session = P2PSession(
|
||||
peer_id=peer_id,
|
||||
local_id=self._local_id,
|
||||
transport=self._data,
|
||||
local_block_len=self._data.block_len,
|
||||
conn=conn,
|
||||
)
|
||||
self._sessions[peer_id] = session
|
||||
return session
|
||||
|
||||
def _accept_new_peers(self, new_connections: Sequence[ControlConnection]) -> None:
|
||||
for conn in new_connections:
|
||||
logger.info(
|
||||
"P2P %s: accepting incoming connection from %s",
|
||||
self._local_id,
|
||||
conn.peer_id,
|
||||
)
|
||||
try:
|
||||
existing = self._sessions.get(conn.peer_id)
|
||||
if existing is not None:
|
||||
raise ValueError(f"duplicate connection from {conn.peer_id}")
|
||||
self._sessions[conn.peer_id] = P2PSession(
|
||||
peer_id=conn.peer_id,
|
||||
local_id=self._local_id,
|
||||
transport=self._data,
|
||||
local_block_len=self._data.block_len,
|
||||
conn=conn,
|
||||
)
|
||||
logger.info(
|
||||
"P2P %s: created connected session for %s",
|
||||
self._local_id,
|
||||
conn.peer_id,
|
||||
)
|
||||
except (ValueError, KeyError, TypeError, AssertionError) as exc:
|
||||
logger.error("P2P %s: rejecting peer: %s", self._local_id, exc)
|
||||
conn.close()
|
||||
|
||||
def _reap_dead_sessions(self) -> None:
|
||||
# Reap connected sessions whose connection died — peer is gone.
|
||||
# Stranded prefiller-side stores are no longer tracked through a
|
||||
# session (they live in _unbound_stores keyed by kv_request_id);
|
||||
# _reap_unbound_stores handles their timeout independently.
|
||||
dead: list[str] | None = None
|
||||
for pid, s in self._sessions.items():
|
||||
if s.connected and not s.alive:
|
||||
if dead is None:
|
||||
dead = []
|
||||
dead.append(pid)
|
||||
if dead is None:
|
||||
return
|
||||
for pid in dead:
|
||||
session = self._sessions.pop(pid)
|
||||
# Purge any kv_request_id → session entries pointing at this
|
||||
# session so subsequent submit_stores fall back to the unbound
|
||||
# path (which will time out into failure if no peer rebinds).
|
||||
stale_kv_ids = [
|
||||
kid for kid, s in self._kv_to_session.items() if s is session
|
||||
]
|
||||
for kid in stale_kv_ids:
|
||||
del self._kv_to_session[kid]
|
||||
failed_loads, failed_stores = session.close()
|
||||
for job_id, kv_request_id in failed_loads:
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
self._failed_req_ids.add(kv_request_id)
|
||||
for job_id in failed_stores:
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
self._data.remove_remote_peer(pid)
|
||||
logger.warning("P2P %s: peer %s down", self._local_id, pid)
|
||||
|
||||
def _reap_unbound_stores(self) -> None:
|
||||
"""Time out submit_store batches that no peer has ever fetched.
|
||||
|
||||
Walks `_unbound_stores` for entries whose oldest batch is older
|
||||
than `_UNBOUND_STORE_TIMEOUT_S`. Drops the kv_request_id, surfaces
|
||||
every batched job as failed, and adds the id to `_failed_req_ids`
|
||||
so a late inbound FetchMsg short-circuits to a clean rejection.
|
||||
"""
|
||||
if not self._unbound_stores:
|
||||
return
|
||||
deadline = time.monotonic() - _UNBOUND_STORE_TIMEOUT_S
|
||||
expired: list[str] | None = None
|
||||
for kid, batches in self._unbound_stores.items():
|
||||
# Batches are appended in arrival order, so the head is oldest.
|
||||
if batches and batches[0].submitted_at <= deadline:
|
||||
if expired is None:
|
||||
expired = []
|
||||
expired.append(kid)
|
||||
if expired is None:
|
||||
return
|
||||
for kid in expired:
|
||||
batches = self._unbound_stores.pop(kid)
|
||||
self._failed_req_ids.add(kid)
|
||||
for batch in batches:
|
||||
self._finished_jobs.append(
|
||||
JobResult(job_id=batch.job_id, success=False)
|
||||
)
|
||||
logger.warning(
|
||||
"P2P %s: unbound store kv_request_id=%s timed out after %.0fs "
|
||||
"without a fetch — failing %d job(s)",
|
||||
self._local_id,
|
||||
kid,
|
||||
_UNBOUND_STORE_TIMEOUT_S,
|
||||
len(batches),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Polling
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _poll_once(self) -> None:
|
||||
"""One sweep of the polling work.
|
||||
|
||||
Drains the control transport, polls every session, accumulates
|
||||
their results into ``_finished_jobs``, and reaps any dead sessions.
|
||||
Runs on the scheduler thread.
|
||||
"""
|
||||
new_connections = self._control.poll()
|
||||
if new_connections:
|
||||
logger.info(
|
||||
"P2P %s: _poll_once got %d new connection(s): %s",
|
||||
self._local_id,
|
||||
len(new_connections),
|
||||
[c.peer_id for c in new_connections],
|
||||
)
|
||||
|
||||
self._accept_new_peers(new_connections)
|
||||
|
||||
for session in self._sessions.values():
|
||||
result = session.poll()
|
||||
for lr in result.loads:
|
||||
self._finished_jobs.append(
|
||||
JobResult(job_id=lr.job_id, success=lr.success)
|
||||
)
|
||||
if not lr.success:
|
||||
self._failed_req_ids.add(lr.kv_request_id)
|
||||
for sr in result.stores:
|
||||
self._finished_jobs.append(
|
||||
JobResult(job_id=sr.job_id, success=sr.success)
|
||||
)
|
||||
# Bind kv_request_id → session for any FetchMsg this tick and
|
||||
# replay any submit_store batches parked while no peer was
|
||||
# asking. ServerRole.on_fetch already recorded the demand
|
||||
# inline in dispatch, so the replayed add_stored_blocks calls
|
||||
# match that demand and submit transfers immediately.
|
||||
for kv_request_id in result.new_fetch_ids:
|
||||
self._kv_to_session[kv_request_id] = session
|
||||
for batch in self._unbound_stores.pop(kv_request_id, ()):
|
||||
session.add_stored_blocks(
|
||||
kv_request_id, batch.keys, batch.block_ids, batch.job_id
|
||||
)
|
||||
|
||||
self._reap_dead_sessions()
|
||||
self._reap_unbound_stores()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
self._drain_inflight_for_shutdown()
|
||||
for session in self._sessions.values():
|
||||
session.close()
|
||||
self._sessions.clear()
|
||||
self._kv_to_session.clear()
|
||||
# Surface buffered store jobs as failed so the engine doesn't
|
||||
# leak them; the manager is going away after this call.
|
||||
for batches in self._unbound_stores.values():
|
||||
for batch in batches:
|
||||
self._finished_jobs.append(
|
||||
JobResult(job_id=batch.job_id, success=False)
|
||||
)
|
||||
self._unbound_stores.clear()
|
||||
self._control.close()
|
||||
self._data.close()
|
||||
|
||||
def _drain_inflight_for_shutdown(self) -> None:
|
||||
"""Best-effort drain of inflight transfers before closing _data.
|
||||
|
||||
Mirrors session._drain_abort but as a single bounded loop. Collects
|
||||
inflight transfer_ids from each session, repeatedly calls
|
||||
_data.cancel(..., mode="wait") and _data.poll() so handles can
|
||||
surface as done/failed, and falls back to mode="immediate" once
|
||||
_SHUTDOWN_DRAIN_TIMEOUT_S elapses so a wedged peer can't hang us.
|
||||
"""
|
||||
ids = [tid for s in self._sessions.values() for tid in s._server._inflight]
|
||||
if not ids:
|
||||
return
|
||||
deadline = time.monotonic() + _SHUTDOWN_DRAIN_TIMEOUT_S
|
||||
still: list[int] = ids
|
||||
while still and time.monotonic() < deadline:
|
||||
still = list(self._data.cancel(still, mode="wait"))
|
||||
if not still:
|
||||
break
|
||||
# poll() advances NIXL handle state so the next wait-cancel
|
||||
# has a chance to release the handles.
|
||||
self._data.poll()
|
||||
time.sleep(_DRAIN_SLEEP_S)
|
||||
if still:
|
||||
logger.warning(
|
||||
"P2P %s: shutdown drain timed out after %.1fs with %d "
|
||||
"transfers still inflight — force-cancelling",
|
||||
self._local_id,
|
||||
_SHUTDOWN_DRAIN_TIMEOUT_S,
|
||||
len(still),
|
||||
)
|
||||
self._data.cancel(still, mode="immediate")
|
||||
@@ -0,0 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.session import (
|
||||
LoadResult,
|
||||
P2PSession,
|
||||
SessionPollResult,
|
||||
StoreResult,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"LoadResult",
|
||||
"P2PSession",
|
||||
"SessionPollResult",
|
||||
"StoreResult",
|
||||
]
|
||||
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Client-role state machine for a single peer session.
|
||||
|
||||
Handles outgoing fetch requests, abort-on-timeout, abort-ack timeout,
|
||||
and produces ``LoadResult`` for completed loads. The session coordinator
|
||||
parses wire messages and dispatches typed arguments here; this module
|
||||
never touches ``ControlConnection`` directly — it emits via the ``send``
|
||||
callback injected by the coordinator (which gates on ConnectAck).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
|
||||
TYPE_KEY,
|
||||
AbortFetchMsg,
|
||||
FetchMsg,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.tiering.base import JobId
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_LOAD_TIMEOUT_S = 30.0
|
||||
_ABORT_ACK_TIMEOUT_S = 10.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _InboundRequestState:
|
||||
"""Client-role state for a single load request."""
|
||||
|
||||
job_id: int # opaque ID assigned by the manager to this load request
|
||||
kv_request_id: str
|
||||
submitted_at: float
|
||||
aborted_at: float | None = None
|
||||
|
||||
|
||||
class LoadResult(NamedTuple):
|
||||
"""Result from a session poll, client side."""
|
||||
|
||||
job_id: int
|
||||
kv_request_id: str
|
||||
success: bool
|
||||
|
||||
|
||||
class ClientRole:
|
||||
"""Client-side load state machine for one peer session.
|
||||
|
||||
The coordinator owns the connection and the send-gating; this role
|
||||
is given a ``send`` callback and a ``peer_id`` for log messages and
|
||||
is otherwise self-contained.
|
||||
"""
|
||||
|
||||
def __init__(self, peer_id: str, send: Callable[[dict], None]) -> None:
|
||||
self._peer_id = peer_id
|
||||
self._send = send
|
||||
self._inbound: dict[str, _InboundRequestState] = {}
|
||||
self._completed_loads: list[LoadResult] = []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def request_blocks(
|
||||
self,
|
||||
job_id: JobId,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[bytes],
|
||||
block_ids: Sequence[int],
|
||||
send_ready: bool,
|
||||
) -> None:
|
||||
"""Register a load request and send the FetchMsg."""
|
||||
logger.debug(
|
||||
"P2PSession %s: request_blocks job_id=%d kv_request_id=%s "
|
||||
"blocks=%d ready=%s",
|
||||
self._peer_id,
|
||||
job_id,
|
||||
kv_request_id,
|
||||
len(block_ids),
|
||||
send_ready,
|
||||
)
|
||||
self._inbound[kv_request_id] = _InboundRequestState(
|
||||
job_id=job_id,
|
||||
kv_request_id=kv_request_id,
|
||||
submitted_at=time.monotonic(),
|
||||
)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.KV_REQUEST_ID: kv_request_id,
|
||||
FetchMsg.BLOCK_HASHES: list(keys),
|
||||
FetchMsg.BLOCK_INDEXES: [int(idx) for idx in block_ids],
|
||||
}
|
||||
)
|
||||
|
||||
def cancel(self, kv_request_id: str) -> None:
|
||||
"""Cancel a pending load. Sends AbortFetchMsg if still active."""
|
||||
req = self._inbound.pop(kv_request_id, None)
|
||||
if req is not None and req.aborted_at is None:
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: kv_request_id,
|
||||
}
|
||||
)
|
||||
|
||||
def on_transfer_done(self, kv_request_id: str, success: bool) -> None:
|
||||
"""Handle a TransferDoneMsg from the peer."""
|
||||
req = self._inbound.pop(kv_request_id, None)
|
||||
if req is not None:
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=req.job_id,
|
||||
kv_request_id=kv_request_id,
|
||||
success=success,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# No matching _inbound entry: either a duplicate
|
||||
# transfer_done from the peer (protocol violation) or a
|
||||
# benign race with a local cancel/abort/timeout that
|
||||
# already popped the entry. We don't track terminated ids,
|
||||
# so we can't tell — log so it's findable.
|
||||
logger.warning(
|
||||
"P2PSession %s: transfer_done for unknown kv_request_id=%s "
|
||||
"(duplicate from peer, or raced with local cancel/timeout)",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
)
|
||||
|
||||
def on_abort_ack(self, kv_request_id: str) -> None:
|
||||
"""Handle an AbortAckMsg from the peer."""
|
||||
req = self._inbound.pop(kv_request_id, None)
|
||||
if req is not None:
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=req.job_id,
|
||||
kv_request_id=kv_request_id,
|
||||
success=False,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# See on_transfer_done: same ambiguity (duplicate ack
|
||||
# vs. raced with local cancel/timeout that already popped).
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_ack for unknown kv_request_id=%s "
|
||||
"(duplicate from peer, or raced with local cancel/timeout)",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
)
|
||||
|
||||
def collect_results(self) -> list[LoadResult]:
|
||||
"""Walk timeouts and drain completed loads.
|
||||
|
||||
Active requests past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg
|
||||
sent and enter the aborting phase. Aborting requests past
|
||||
``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed loads.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
to_remove: list[str] = []
|
||||
for req_id, req in self._inbound.items():
|
||||
if req.aborted_at is None:
|
||||
if now - req.submitted_at >= _LOAD_TIMEOUT_S:
|
||||
req.aborted_at = now
|
||||
logger.warning(
|
||||
"P2PSession %s: %s timed out, sending abort",
|
||||
self._peer_id,
|
||||
req_id,
|
||||
)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: req_id,
|
||||
}
|
||||
)
|
||||
else:
|
||||
if now - req.aborted_at >= _ABORT_ACK_TIMEOUT_S:
|
||||
to_remove.append(req_id)
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=req.job_id,
|
||||
kv_request_id=req_id,
|
||||
success=False,
|
||||
)
|
||||
)
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_ack timed out for kv_request_id=%s",
|
||||
self._peer_id,
|
||||
req_id,
|
||||
)
|
||||
for req_id in to_remove:
|
||||
self._inbound.pop(req_id)
|
||||
|
||||
results = self._completed_loads
|
||||
self._completed_loads = []
|
||||
return results
|
||||
|
||||
def close(self) -> list[tuple[int, str]]:
|
||||
"""Tear down. Returns ``(job_id, kv_request_id)`` for pending loads."""
|
||||
failed = [(req.job_id, req.kv_request_id) for req in self._inbound.values()]
|
||||
self._inbound.clear()
|
||||
self._completed_loads.clear()
|
||||
return failed
|
||||
@@ -0,0 +1,247 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
P2P KV cache sharing protocol constants and documentation.
|
||||
|
||||
Protocol Overview
|
||||
=================
|
||||
|
||||
Two session types communicate over a bidirectional message channel:
|
||||
- P2PClientSession (requests blocks from a server)
|
||||
- P2PServerSession (serves blocks to a client)
|
||||
|
||||
Connection Lifecycle
|
||||
--------------------
|
||||
|
||||
1. Client opens a connection and sends ConnectMsg with its identity,
|
||||
RDMA metadata, memory layout, and config fingerprint.
|
||||
2. Server validates block_len and config_fingerprint, registers the
|
||||
RDMA peer, and replies with ConnectAckMsg.
|
||||
3. Client receives ConnectAckMsg and transitions to ready state
|
||||
(flushes any queued messages).
|
||||
4. Either side may send DisconnectMsg to gracefully close.
|
||||
|
||||
Block Transfer Flow (happy path)
|
||||
---------------------------------
|
||||
|
||||
1. Client sends FetchMsg with a kv_request_id and lists of
|
||||
block keys + remote indexes where it wants the data written.
|
||||
2. Server matches requested blocks against locally stored blocks:
|
||||
- Blocks already available are transferred immediately via RDMA.
|
||||
- Blocks not yet available are recorded as "demanded" and
|
||||
transferred when the server later stores them.
|
||||
3. When all blocks for a kv_request_id are transferred, the server
|
||||
sends TransferDoneMsg (success=True) to the client.
|
||||
4. Client reports the load job as complete.
|
||||
|
||||
Abort Flow (timeout path)
|
||||
--------------------------
|
||||
|
||||
1. If the client times out waiting for TransferDoneMsg, it sends
|
||||
AbortFetchMsg to cancel the request.
|
||||
2. Server cancels inflight transfers for that kv_request_id and
|
||||
replies with AbortAckMsg.
|
||||
3. Client receives AbortAckMsg and reports the load job as failed.
|
||||
4. If AbortAckMsg itself times out, the client fails the job anyway.
|
||||
|
||||
Message Format
|
||||
--------------
|
||||
|
||||
All messages are dicts serialized with msgpack. Every message has a
|
||||
TYPE_KEY key identifying its type. Additional fields depend on the
|
||||
message type (see per-message class docstrings below).
|
||||
|
||||
Security
|
||||
--------
|
||||
|
||||
Sessions wrap all incoming message handling in try/except to guard
|
||||
against malformed messages from adversarial or buggy peers. Invalid
|
||||
messages are logged and dropped without crashing the session.
|
||||
|
||||
The config_fingerprint field in ConnectMsg ensures peers have
|
||||
compatible model configurations (model, dtype, block sizes). Mismatches
|
||||
are rejected during the handshake.
|
||||
"""
|
||||
|
||||
TYPE_KEY = "type"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require(msg: dict, key: str, typ: type, *, name: str = "") -> None:
|
||||
"""Raise ValueError if msg[key] is missing or not isinstance(typ)."""
|
||||
val = msg.get(key)
|
||||
if not isinstance(val, typ):
|
||||
label = name or key
|
||||
raise ValueError(f"{label}: expected {typ.__name__}, got {type(val).__name__}")
|
||||
|
||||
|
||||
def _require_pos_int(msg: dict, key: str, *, name: str = "") -> None:
|
||||
"""Raise ValueError if msg[key] is not a positive int."""
|
||||
val = msg.get(key)
|
||||
if not isinstance(val, int) or val <= 0:
|
||||
label = name or key
|
||||
raise ValueError(f"{label}: expected positive int, got {val!r}")
|
||||
|
||||
|
||||
def _require_non_neg_int(msg: dict, key: str, *, name: str = "") -> None:
|
||||
"""Raise ValueError if msg[key] is not a non-negative int."""
|
||||
val = msg.get(key)
|
||||
if not isinstance(val, int) or val < 0:
|
||||
label = name or key
|
||||
raise ValueError(f"{label}: expected non-negative int, got {val!r}")
|
||||
|
||||
|
||||
def _require_list(msg: dict, key: str, *, name: str = "") -> None:
|
||||
"""Raise ValueError if msg[key] is not a list."""
|
||||
val = msg.get(key)
|
||||
if not isinstance(val, list):
|
||||
label = name or key
|
||||
raise ValueError(f"{label}: expected list, got {type(val).__name__}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Message classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ConnectMsg:
|
||||
"""Client → Server: initial handshake request.
|
||||
|
||||
Fields:
|
||||
PEER_ID: Local peer identity string.
|
||||
AGENT_METADATA: RDMA agent metadata (opaque bytes).
|
||||
BASE_ADDR: Base memory address of the KV block region.
|
||||
NUM_BLOCKS: Number of blocks in the KV block region.
|
||||
BLOCK_LEN: Size in bytes of each block (must match between peers).
|
||||
CONFIG_FINGERPRINT: SHA-256 prefix of the model configuration.
|
||||
Peers with different fingerprints are incompatible.
|
||||
"""
|
||||
|
||||
TYPE = "connect"
|
||||
PEER_ID = "peer_id"
|
||||
AGENT_METADATA = "agent_metadata"
|
||||
BASE_ADDR = "base_addr"
|
||||
NUM_BLOCKS = "num_blocks"
|
||||
BLOCK_LEN = "block_len"
|
||||
CONFIG_FINGERPRINT = "config_fingerprint"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, ConnectMsg.PEER_ID, str)
|
||||
_require(msg, ConnectMsg.AGENT_METADATA, bytes)
|
||||
_require_non_neg_int(msg, ConnectMsg.BASE_ADDR)
|
||||
_require_pos_int(msg, ConnectMsg.NUM_BLOCKS)
|
||||
_require_pos_int(msg, ConnectMsg.BLOCK_LEN)
|
||||
|
||||
|
||||
class ConnectAckMsg:
|
||||
"""Server → Client: handshake acknowledgement.
|
||||
|
||||
Fields:
|
||||
PEER_ID: Server's peer identity string.
|
||||
"""
|
||||
|
||||
TYPE = "connect_ack"
|
||||
PEER_ID = "peer_id"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, ConnectAckMsg.PEER_ID, str)
|
||||
|
||||
|
||||
class DisconnectMsg:
|
||||
"""Either → Either: graceful connection close.
|
||||
|
||||
No additional fields beyond TYPE_KEY.
|
||||
"""
|
||||
|
||||
TYPE = "disconnect"
|
||||
|
||||
|
||||
class FetchMsg:
|
||||
"""Client → Server: request blocks by key.
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: Identifies this block transfer request.
|
||||
BLOCK_HASHES: List of block keys (OffloadKey bytes).
|
||||
BLOCK_INDEXES: List of remote block indexes (same length as BLOCK_HASHES).
|
||||
"""
|
||||
|
||||
TYPE = "fetch"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
BLOCK_HASHES = "block_hashes"
|
||||
BLOCK_INDEXES = "block_indexes"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, FetchMsg.KV_REQUEST_ID, str)
|
||||
_require_list(msg, FetchMsg.BLOCK_HASHES)
|
||||
_require_list(msg, FetchMsg.BLOCK_INDEXES)
|
||||
hashes = msg[FetchMsg.BLOCK_HASHES]
|
||||
indexes = msg[FetchMsg.BLOCK_INDEXES]
|
||||
if len(hashes) != len(indexes):
|
||||
raise ValueError(
|
||||
f"block_hashes/block_indexes length mismatch: "
|
||||
f"{len(hashes)} vs {len(indexes)}"
|
||||
)
|
||||
for idx in indexes:
|
||||
if not isinstance(idx, int) or idx < 0:
|
||||
raise ValueError(f"block_indexes: invalid index {idx!r}")
|
||||
|
||||
|
||||
class TransferDoneMsg:
|
||||
"""Server → Client: all blocks transferred for a request.
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: The request that completed.
|
||||
SUCCESS: Whether the transfer completed successfully.
|
||||
"""
|
||||
|
||||
TYPE = "transfer_done"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
SUCCESS = "success"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, TransferDoneMsg.KV_REQUEST_ID, str)
|
||||
_require(msg, TransferDoneMsg.SUCCESS, bool)
|
||||
|
||||
|
||||
class AbortFetchMsg:
|
||||
"""Client → Server: cancel a pending request.
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: The request to cancel.
|
||||
"""
|
||||
|
||||
TYPE = "abort_fetch"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, AbortFetchMsg.KV_REQUEST_ID, str)
|
||||
|
||||
|
||||
class AbortAckMsg:
|
||||
"""Server → Client: acknowledge cancellation.
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: The request that was cancelled.
|
||||
"""
|
||||
|
||||
TYPE = "abort_ack"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, AbortAckMsg.KV_REQUEST_ID, str)
|
||||
@@ -0,0 +1,601 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Server-role state machine for a single peer session.
|
||||
|
||||
Owns block matching (supply vs. demand), inflight RDMA transfers, store-job
|
||||
timeouts, abort-drain, and produces ``StoreResult`` for completed stores.
|
||||
The session coordinator parses wire messages and dispatches typed
|
||||
arguments here; this module never touches ``ControlConnection`` directly
|
||||
— it emits via the ``send`` callback injected by the coordinator (which
|
||||
gates on ConnectAck).
|
||||
|
||||
Protocol violations the role can detect (today: duplicate ``FetchMsg``
|
||||
for the same ``kv_request_id``) are surfaced as ``ValueError`` so the
|
||||
coordinator's ``_dispatch_message`` can reuse its existing
|
||||
``_protocol_error`` path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import OffloadKey
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
|
||||
TYPE_KEY,
|
||||
AbortAckMsg,
|
||||
TransferDoneMsg,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.tiering.base import JobId
|
||||
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_STORE_TIMEOUT_S = 30.0
|
||||
_CANCEL_DRAIN_TIMEOUT_S = 10.0
|
||||
|
||||
|
||||
class StoreResult(NamedTuple):
|
||||
"""Result from a session poll, server side."""
|
||||
|
||||
job_id: int
|
||||
success: bool
|
||||
|
||||
|
||||
class _InflightXfer(NamedTuple):
|
||||
"""Metadata for a single inflight RDMA transfer, keyed by transfer_id."""
|
||||
|
||||
kv_request_id: str
|
||||
block_count: int
|
||||
# The set of store job IDs that contributed blocks to this transfer.
|
||||
job_ids: set[int]
|
||||
|
||||
|
||||
class _MatchResult(NamedTuple):
|
||||
"""Result of block matching: pairs ready for transfer."""
|
||||
|
||||
local_idxs: list[int]
|
||||
remote_idxs: list[int]
|
||||
# The set of store job IDs that contributed blocks
|
||||
job_ids: set[int]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _OutboundRequestState:
|
||||
"""Server-role state for a single peer fetch request.
|
||||
|
||||
The owning ``kv_request_id`` is the dict key in
|
||||
``ServerRole._outbound`` and is not duplicated on the value.
|
||||
"""
|
||||
|
||||
demand_received: bool = False
|
||||
available: dict[OffloadKey, tuple[int, int]] = field(
|
||||
default_factory=dict
|
||||
) # key → (job_id, local_block_idx): blocks we have, awaiting demand
|
||||
demanded: dict[OffloadKey, int] = field(
|
||||
default_factory=dict
|
||||
) # key → remote_block_idx: blocks peer wants, awaiting supply
|
||||
remaining: int = 0 # blocks that need to be transferred to client
|
||||
finishing: bool = False # Signal finish request ASAP
|
||||
# Job IDs that submit_store'd blocks for this request and have not
|
||||
# yet emitted a StoreResult. The terminal-finalize helper drains
|
||||
# this set; poll-done and poll-failed discard entries as their
|
||||
# StoreResults fire.
|
||||
pending_job_ids: set[int] = field(default_factory=set)
|
||||
|
||||
def add_stored_blocks(
|
||||
self,
|
||||
block_hashes: Sequence[OffloadKey],
|
||||
block_ids: Sequence[int],
|
||||
job_id: int,
|
||||
) -> _MatchResult:
|
||||
"""Add locally-stored blocks. Returns matched pairs."""
|
||||
self.pending_job_ids.add(job_id)
|
||||
local_idxs: list[int] = []
|
||||
remote_idxs: list[int] = []
|
||||
for block_hash, local_idx in zip(block_hashes, block_ids):
|
||||
remote_idx = self.demanded.pop(block_hash, None)
|
||||
if remote_idx is not None:
|
||||
local_idxs.append(local_idx)
|
||||
remote_idxs.append(remote_idx)
|
||||
else:
|
||||
self.available[block_hash] = (job_id, local_idx)
|
||||
return _MatchResult(
|
||||
local_idxs=local_idxs,
|
||||
remote_idxs=remote_idxs,
|
||||
job_ids={job_id} if local_idxs else set(),
|
||||
)
|
||||
|
||||
def add_fetch_demand(
|
||||
self,
|
||||
block_hashes: Sequence[OffloadKey],
|
||||
block_indexes: Sequence[int],
|
||||
) -> _MatchResult:
|
||||
"""Register the peer's fetch demand. Returns matched pairs."""
|
||||
self.demand_received = True
|
||||
self.remaining = len(block_hashes)
|
||||
|
||||
local_idxs: list[int] = []
|
||||
remote_idxs: list[int] = []
|
||||
job_ids: set[int] = set()
|
||||
for block_hash, remote_idx in zip(block_hashes, block_indexes):
|
||||
stored_entry = self.available.pop(block_hash, None)
|
||||
if stored_entry is not None:
|
||||
stored_job_id, local_idx = stored_entry
|
||||
local_idxs.append(local_idx)
|
||||
remote_idxs.append(remote_idx)
|
||||
job_ids.add(stored_job_id)
|
||||
else:
|
||||
self.demanded[block_hash] = remote_idx
|
||||
return _MatchResult(
|
||||
local_idxs=local_idxs,
|
||||
remote_idxs=remote_idxs,
|
||||
job_ids=job_ids,
|
||||
)
|
||||
|
||||
|
||||
class ServerRole:
|
||||
"""Server-side store/serve state machine for one peer session.
|
||||
|
||||
The coordinator owns the connection and the send-gating; this role
|
||||
is given a ``send`` callback, the ``DataTransport``, and the
|
||||
``peer_id`` for transport calls and log messages.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
peer_id: str,
|
||||
transport: DataTransport,
|
||||
send: Callable[[dict], None],
|
||||
) -> None:
|
||||
self._peer_id = peer_id
|
||||
self._transport = transport
|
||||
self._send = send
|
||||
|
||||
self._outbound: dict[str, _OutboundRequestState] = {}
|
||||
# transfer_id → xfer. Mutate ONLY via _inflight_add / _inflight_pop
|
||||
# so the per-request count below stays in sync.
|
||||
self._inflight: dict[int, _InflightXfer] = {}
|
||||
# kv_request_id → number of entries in _inflight for that id.
|
||||
# Kept in sync with _inflight; entries that hit zero are removed
|
||||
# so `kv_request_id in self._inflight_per_req` is an exact
|
||||
# "has any inflight transfer" predicate (O(1) replacement for
|
||||
# the previous O(N) scan).
|
||||
self._inflight_per_req: dict[str, int] = {}
|
||||
self._store_jobs: dict[int, float] = {} # job_id → submitted_at
|
||||
self._pending_aborts: dict[str, float] = {} # kv_request_id → start
|
||||
# StoreResults queued by _finalize_outbound for the next poll
|
||||
# tick to surface. Mirrors the deferred-result pattern used for
|
||||
# load timeouts.
|
||||
self._pending_store_results: list[StoreResult] = []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_stored_blocks(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[OffloadKey],
|
||||
block_ids: Sequence[int],
|
||||
job_id: JobId,
|
||||
) -> None:
|
||||
"""New blocks stored locally — match against pending fetch demand."""
|
||||
self._store_jobs[job_id] = time.monotonic()
|
||||
req = self._outbound.setdefault(kv_request_id, _OutboundRequestState())
|
||||
result = req.add_stored_blocks(keys, block_ids, job_id)
|
||||
if result.local_idxs and req.demand_received:
|
||||
self._submit_transfer(kv_request_id, result)
|
||||
|
||||
def on_fetch(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
block_hashes: Sequence[OffloadKey],
|
||||
block_indexes: Sequence[int],
|
||||
) -> None:
|
||||
"""Handle a FetchMsg from the peer.
|
||||
|
||||
Raises ``ValueError`` on a duplicate fetch for the same
|
||||
``kv_request_id``; the coordinator's dispatch loop turns that
|
||||
into a protocol-error disconnect.
|
||||
"""
|
||||
logger.debug(
|
||||
"P2PSession %s: fetch RECEIVED kv_request_id=%s blocks=%d",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
len(block_hashes),
|
||||
)
|
||||
existing = self._outbound.get(kv_request_id)
|
||||
if existing is not None and existing.demand_received:
|
||||
# A second fetch for the same kv_request_id would overwrite
|
||||
# `remaining` and leak inflight bookkeeping. Treat as a
|
||||
# protocol violation.
|
||||
raise ValueError(f"duplicate fetch for kv_request_id={kv_request_id}")
|
||||
req = self._outbound.setdefault(kv_request_id, _OutboundRequestState())
|
||||
result = req.add_fetch_demand(block_hashes, block_indexes)
|
||||
if result.local_idxs:
|
||||
self._submit_transfer(kv_request_id, result)
|
||||
# Prefiller-first mode: finish_request may have run before
|
||||
# fetch arrived. If so, finalize once we know what was
|
||||
# demanded — fully satisfied → success, else early-fail.
|
||||
if req.finishing and not self._has_inflight_for(kv_request_id):
|
||||
self._finalize_outbound(kv_request_id)
|
||||
|
||||
def on_abort_fetch(self, kv_request_id: str) -> None:
|
||||
"""Handle an AbortFetchMsg from the peer."""
|
||||
# Abort for an unknown id may be a benign race/duplicate or a
|
||||
# real protocol violation; we don't track completed ids, so warn.
|
||||
if kv_request_id not in self._outbound and not self._has_inflight_for(
|
||||
kv_request_id
|
||||
):
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_fetch for unknown kv_request_id=%s "
|
||||
"(no outbound or inflight state); benign race or stale",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
)
|
||||
# Idempotent: receiving AbortFetchMsg again before we've sent the
|
||||
# ack just triggers another drain attempt without resetting the
|
||||
# deadline.
|
||||
self._pending_aborts.setdefault(kv_request_id, time.monotonic())
|
||||
self._drain_abort(kv_request_id)
|
||||
|
||||
def finish(self, kv_request_id: str) -> None:
|
||||
"""Mark an outbound request finishing.
|
||||
|
||||
No more submit_store calls will arrive for this id. Any blocks
|
||||
the peer demanded but we never stored will never come; tell the
|
||||
peer to stop waiting (TransferDoneMsg success=False) instead of
|
||||
letting it hit _LOAD_TIMEOUT_S.
|
||||
|
||||
If the decoder hasn't sent fetch yet (no demand received),
|
||||
defer — on_fetch will finalize once demand arrives.
|
||||
|
||||
If inflight transfers exist for this id, defer — the last
|
||||
completing transfer in collect_results will fire the message.
|
||||
"""
|
||||
req = self._outbound.get(kv_request_id)
|
||||
if req is None:
|
||||
return
|
||||
req.finishing = True
|
||||
if not req.demand_received:
|
||||
return
|
||||
if self._has_inflight_for(kv_request_id):
|
||||
return
|
||||
# Remaining > 0 here: if it had hit 0, the poll-done success
|
||||
# branch would have already popped _outbound and we'd have
|
||||
# returned at `req is None` above. Helper derives success from
|
||||
# remaining and emits StoreResult(success=False) for any
|
||||
# leftover pending jobs.
|
||||
self._finalize_outbound(kv_request_id)
|
||||
|
||||
def collect_results(self) -> list[StoreResult]:
|
||||
"""Drain timeouts, deferred results, and transport completions."""
|
||||
results: list[StoreResult] = self._timeout_pending_store_jobs()
|
||||
|
||||
if self._pending_store_results:
|
||||
results.extend(self._pending_store_results)
|
||||
self._pending_store_results.clear()
|
||||
|
||||
poll_result = self._transport.poll()
|
||||
|
||||
for tid in poll_result.done:
|
||||
xfer = self._inflight_pop(tid)
|
||||
if xfer is None:
|
||||
# Bug signal: transport reported a transfer we have no
|
||||
# bookkeeping for. Likely a double-completion in the
|
||||
# transport or a stale removal in the session. The
|
||||
# attached job(s) still live in _store_jobs and will be
|
||||
# surfaced as failures by _timeout_pending_store_jobs
|
||||
# after _STORE_TIMEOUT_S, but log loudly so the
|
||||
# underlying bug is findable.
|
||||
logger.error(
|
||||
"P2PSession %s: transport reported done for unknown "
|
||||
"transfer_id=%d; attached job(s) will fail via "
|
||||
"store-timeout instead of completing now",
|
||||
self._peer_id,
|
||||
tid,
|
||||
)
|
||||
continue
|
||||
req = self._outbound.get(xfer.kv_request_id)
|
||||
for job_id in xfer.job_ids:
|
||||
if self._store_jobs.pop(job_id, None) is None:
|
||||
# Already reported (timeout, cancellation, etc.) —
|
||||
# don't double-emit a contradictory success result.
|
||||
continue
|
||||
results.append(StoreResult(job_id=job_id, success=True))
|
||||
if req is not None:
|
||||
req.pending_job_ids.discard(job_id)
|
||||
if req is not None and req.demand_received:
|
||||
req.remaining -= xfer.block_count
|
||||
assert req.remaining >= 0, (
|
||||
f"remaining went negative for kv_request_id={xfer.kv_request_id}"
|
||||
)
|
||||
if req.remaining == 0:
|
||||
self._finalize_outbound(xfer.kv_request_id, success=True)
|
||||
elif req.finishing and not self._has_inflight_for(xfer.kv_request_id):
|
||||
self._finalize_outbound(xfer.kv_request_id, success=False)
|
||||
|
||||
failed_kv_request_ids: set[str] | None = None
|
||||
for tid in poll_result.failed:
|
||||
xfer = self._inflight_pop(tid)
|
||||
if xfer is None:
|
||||
# See the matching error log in the done branch above.
|
||||
logger.error(
|
||||
"P2PSession %s: transport reported failed for unknown "
|
||||
"transfer_id=%d; attached job(s) will fail via "
|
||||
"store-timeout instead of completing now",
|
||||
self._peer_id,
|
||||
tid,
|
||||
)
|
||||
continue
|
||||
if failed_kv_request_ids is None:
|
||||
failed_kv_request_ids = set()
|
||||
failed_kv_request_ids.add(xfer.kv_request_id)
|
||||
req_for_xfer = self._outbound.get(xfer.kv_request_id)
|
||||
for job_id in xfer.job_ids:
|
||||
if self._store_jobs.pop(job_id, None) is None:
|
||||
# Already reported (timeout, cancellation, etc.) —
|
||||
# don't double-emit.
|
||||
continue
|
||||
results.append(StoreResult(job_id=job_id, success=False))
|
||||
if req_for_xfer is not None:
|
||||
req_for_xfer.pending_job_ids.discard(job_id)
|
||||
req = self._outbound.pop(xfer.kv_request_id, None)
|
||||
if req is not None and req.demand_received:
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.KV_REQUEST_ID: xfer.kv_request_id,
|
||||
TransferDoneMsg.SUCCESS: False,
|
||||
}
|
||||
)
|
||||
|
||||
# Cancel other inflight for the same failed kv_request_ids
|
||||
if failed_kv_request_ids:
|
||||
ids_to_cancel = [
|
||||
tid
|
||||
for tid, xfer in self._inflight.items()
|
||||
if xfer.kv_request_id in failed_kv_request_ids
|
||||
]
|
||||
for tid in ids_to_cancel:
|
||||
self._inflight_pop(tid)
|
||||
self._transport.cancel(ids_to_cancel)
|
||||
|
||||
return results
|
||||
|
||||
def collect_idle_timeouts(self) -> list[StoreResult]:
|
||||
"""Run only the store-job timeout sweep.
|
||||
|
||||
Used by the coordinator's no-conn poll path: a pending session
|
||||
cannot have inflight transfers (no peer registered yet), so we
|
||||
skip the transport poll and the deferred-result drain.
|
||||
"""
|
||||
return self._timeout_pending_store_jobs()
|
||||
|
||||
def drain_pending_aborts(self) -> None:
|
||||
"""Re-attempt every parked abort once per poll tick."""
|
||||
if not self._pending_aborts:
|
||||
return
|
||||
for kv_request_id in list(self._pending_aborts):
|
||||
self._drain_abort(kv_request_id)
|
||||
|
||||
def close(self) -> list[int]:
|
||||
"""Tear down. Cancels inflight, returns failed store job ids."""
|
||||
failed_stores = list(self._store_jobs.keys())
|
||||
self._store_jobs.clear()
|
||||
if self._inflight:
|
||||
self._transport.cancel(list(self._inflight.keys()))
|
||||
self._inflight.clear()
|
||||
self._inflight_per_req.clear()
|
||||
self._outbound.clear()
|
||||
self._pending_aborts.clear()
|
||||
self._pending_store_results.clear()
|
||||
return failed_stores
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal — inflight bookkeeping
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _has_inflight_for(self, kv_request_id: str) -> bool:
|
||||
return kv_request_id in self._inflight_per_req
|
||||
|
||||
def _inflight_add(self, tid: int, xfer: _InflightXfer) -> None:
|
||||
"""Insert an inflight transfer and bump the per-request count."""
|
||||
self._inflight[tid] = xfer
|
||||
self._inflight_per_req[xfer.kv_request_id] = (
|
||||
self._inflight_per_req.get(xfer.kv_request_id, 0) + 1
|
||||
)
|
||||
|
||||
def _inflight_pop(self, tid: int) -> _InflightXfer | None:
|
||||
"""Pop an inflight transfer and decrement the per-request count.
|
||||
|
||||
Removes the per-request entry once the count hits zero so the
|
||||
dict stays bounded and `_has_inflight_for` remains exact.
|
||||
"""
|
||||
xfer = self._inflight.pop(tid, None)
|
||||
if xfer is None:
|
||||
return None
|
||||
new_count = self._inflight_per_req.get(xfer.kv_request_id, 0) - 1
|
||||
if new_count > 0:
|
||||
self._inflight_per_req[xfer.kv_request_id] = new_count
|
||||
else:
|
||||
self._inflight_per_req.pop(xfer.kv_request_id, None)
|
||||
return xfer
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal — finalize / abort drain
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _finalize_outbound(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
success: bool | None = None,
|
||||
) -> None:
|
||||
"""Pop the outbound state and emit terminal results.
|
||||
|
||||
Called when no further work will happen for this kv_request_id
|
||||
on the server side: either request_finish has fired and there
|
||||
are no inflight transfers, or the last inflight just completed
|
||||
while finishing.
|
||||
|
||||
If ``success`` is None, derive it from ``req.remaining == 0``.
|
||||
The same flag is used for both the peer's TransferDoneMsg and
|
||||
the StoreResult(s) emitted for any leftover pending job_ids.
|
||||
"""
|
||||
req = self._outbound.pop(kv_request_id, None)
|
||||
if req is None:
|
||||
return
|
||||
if success is None:
|
||||
success = req.remaining == 0
|
||||
for job_id in req.pending_job_ids:
|
||||
self._store_jobs.pop(job_id, None)
|
||||
self._pending_store_results.append(
|
||||
StoreResult(job_id=job_id, success=success)
|
||||
)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.KV_REQUEST_ID: kv_request_id,
|
||||
TransferDoneMsg.SUCCESS: success,
|
||||
}
|
||||
)
|
||||
|
||||
def _drain_abort(self, kv_request_id: str) -> None:
|
||||
"""One drain attempt for a pending abort.
|
||||
|
||||
Stops accepting more blocks for ``kv_request_id``, then asks the
|
||||
transport to cancel any matching inflight transfers in
|
||||
``mode="wait"``. Sends ``AbortAckMsg`` once nothing remains
|
||||
inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S`` falls back to
|
||||
``mode="immediate"`` and acks anyway.
|
||||
"""
|
||||
self._outbound.pop(kv_request_id, None)
|
||||
ids = [
|
||||
tid
|
||||
for tid, xfer in self._inflight.items()
|
||||
if xfer.kv_request_id == kv_request_id
|
||||
]
|
||||
if not ids:
|
||||
self._finalize_abort(kv_request_id)
|
||||
return
|
||||
|
||||
started_at = self._pending_aborts.get(kv_request_id)
|
||||
expired = (
|
||||
started_at is not None
|
||||
and time.monotonic() - started_at >= _CANCEL_DRAIN_TIMEOUT_S
|
||||
)
|
||||
if expired:
|
||||
for tid in ids:
|
||||
self._inflight_pop(tid)
|
||||
self._transport.cancel(ids, mode="immediate")
|
||||
logger.warning(
|
||||
"P2PSession %s: cancel drain timed out for kv_request_id=%s,"
|
||||
" force-canceled %d transfers",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
len(ids),
|
||||
)
|
||||
self._finalize_abort(kv_request_id)
|
||||
return
|
||||
|
||||
still = self._transport.cancel(ids, mode="wait")
|
||||
# Tids the transport successfully released are gone from its
|
||||
# _inflight; mirror that in session bookkeeping so they don't
|
||||
# block the drain forever waiting for a poll() event that will
|
||||
# never come.
|
||||
still_set = set(still)
|
||||
for tid in ids:
|
||||
if tid not in still_set:
|
||||
self._inflight_pop(tid)
|
||||
if not still:
|
||||
self._finalize_abort(kv_request_id)
|
||||
|
||||
def _finalize_abort(self, kv_request_id: str) -> None:
|
||||
self._pending_aborts.pop(kv_request_id, None)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortAckMsg.TYPE,
|
||||
AbortAckMsg.KV_REQUEST_ID: kv_request_id,
|
||||
}
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal — transfers and store-job timeouts
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _submit_transfer(self, kv_request_id: str, result: _MatchResult) -> None:
|
||||
logger.debug(
|
||||
"P2PSession %s: NIXL write_blocks CALL kv_request_id=%s "
|
||||
"local_idxs=%d remote_idxs=%d",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
len(result.local_idxs),
|
||||
len(result.remote_idxs),
|
||||
)
|
||||
transfer_id = self._transport.write_blocks(
|
||||
self._peer_id, result.local_idxs, result.remote_idxs
|
||||
)
|
||||
if transfer_id is not None:
|
||||
logger.debug(
|
||||
"P2PSession %s: NIXL write_blocks SUBMITTED kv_request_id=%s "
|
||||
"transfer_id=%d blocks=%d",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
transfer_id,
|
||||
len(result.local_idxs),
|
||||
)
|
||||
self._inflight_add(
|
||||
transfer_id,
|
||||
_InflightXfer(
|
||||
kv_request_id=kv_request_id,
|
||||
block_count=len(result.local_idxs),
|
||||
job_ids=result.job_ids,
|
||||
),
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"P2PSession %s: write_blocks failed for %s (%d blocks)",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
len(result.local_idxs),
|
||||
)
|
||||
# The matched blocks were popped from req.demanded /
|
||||
# req.available, but no inflight will satisfy them, so
|
||||
# remaining will never reach 0 on its own. Mark the
|
||||
# request as finishing so the existing terminal paths
|
||||
# clean up: if other inflight is in flight, the last one
|
||||
# to drain will fire _finalize_outbound(success=False)
|
||||
# via the elif branch in collect_results. If
|
||||
# nothing else is in flight, finalize now so the peer
|
||||
# and the local store jobs don't wait for finish_request
|
||||
# or for _STORE_TIMEOUT_S / _LOAD_TIMEOUT_S.
|
||||
req = self._outbound.get(kv_request_id)
|
||||
if req is not None:
|
||||
req.finishing = True
|
||||
if not self._has_inflight_for(kv_request_id):
|
||||
self._finalize_outbound(kv_request_id, success=False)
|
||||
|
||||
def _timeout_pending_store_jobs(self) -> list[StoreResult]:
|
||||
if not self._store_jobs:
|
||||
return []
|
||||
deadline = time.monotonic() - _STORE_TIMEOUT_S
|
||||
timed_out: list[int] | None = None
|
||||
for jid, submitted_at in self._store_jobs.items():
|
||||
if submitted_at <= deadline:
|
||||
if timed_out is None:
|
||||
timed_out = []
|
||||
timed_out.append(jid)
|
||||
if timed_out is None:
|
||||
return []
|
||||
results: list[StoreResult] = []
|
||||
for jid in timed_out:
|
||||
del self._store_jobs[jid]
|
||||
results.append(StoreResult(job_id=jid, success=False))
|
||||
logger.warning("P2PSession %s: store job %d timed out", self._peer_id, jid)
|
||||
return results
|
||||
@@ -0,0 +1,440 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
P2PSession — bidirectional session combining client + server roles.
|
||||
|
||||
A single P2PSession per remote peer handles BOTH directions of the P2P
|
||||
protocol on one ControlConnection: it can request blocks from the peer
|
||||
("client" role, in :mod:`.client`) AND serve blocks to the peer
|
||||
("server" role, in :mod:`.server`). This module is the thin coordinator
|
||||
that owns the connection, the handshake, send-gating, and the message
|
||||
dispatch — each parsed message is forwarded to the corresponding role.
|
||||
|
||||
Wire protocol is unchanged. Both sides advertise their NIXL metadata
|
||||
via ConnectMsg when their session is connected; the peer's ConnectMsg
|
||||
triggers transport.add_remote_peer; ConnectAckMsg confirms the peer
|
||||
received our ConnectMsg, after which queued outgoing messages are flushed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import OffloadKey
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.base import ControlConnection
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.client import ClientRole, LoadResult
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
|
||||
TYPE_KEY,
|
||||
AbortAckMsg,
|
||||
AbortFetchMsg,
|
||||
ConnectAckMsg,
|
||||
ConnectMsg,
|
||||
DisconnectMsg,
|
||||
FetchMsg,
|
||||
TransferDoneMsg,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.server import (
|
||||
ServerRole,
|
||||
StoreResult,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.tiering.base import JobId
|
||||
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Cap on consecutive non-protocol dispatch exceptions before we tear
|
||||
# down the session. Protocol violations (ValueError) disconnect on the
|
||||
# first occurrence; this threshold protects against repeated internal
|
||||
# bugs that may indicate a peer-induced bad state. Reset on any
|
||||
# successful dispatch.
|
||||
_MAX_CONSECUTIVE_DISPATCH_ERRORS = 5
|
||||
|
||||
|
||||
class SessionPollResult(NamedTuple):
|
||||
"""Result of one P2PSession.poll() tick.
|
||||
|
||||
`loads`/`stores` are the same per-role results the manager has always
|
||||
consumed. `new_fetch_ids` reports kv_request_ids whose FetchMsg
|
||||
arrived this tick — the manager uses them to bind kv_request_id →
|
||||
session and replay any submit_store batches parked while no peer had
|
||||
asked yet. Reporting (rather than calling back into the manager
|
||||
mid-dispatch) keeps the dependency strictly top-down.
|
||||
"""
|
||||
|
||||
loads: list[LoadResult]
|
||||
stores: list[StoreResult]
|
||||
new_fetch_ids: list[str]
|
||||
|
||||
|
||||
class P2PSession:
|
||||
"""Bidirectional session — coordinator over ClientRole + ServerRole.
|
||||
|
||||
Lifecycle:
|
||||
- Constructor with conn=None ⇒ pending. Accepts add_stored_blocks
|
||||
but cannot send (used by the prefiller to buffer blocks before
|
||||
the decoder connects).
|
||||
- Constructor with conn != None ⇒ connected. Sends our own ConnectMsg
|
||||
immediately; the peer's ConnectMsg arrives in poll() and is
|
||||
dispatched to _on_connect (which calls transport.add_remote_peer
|
||||
and replies with ConnectAckMsg). Outgoing sends are queued until
|
||||
ConnectAckMsg confirms our metadata reached the peer.
|
||||
- attach_connection(conn) on a pending session ⇒ same as above,
|
||||
starting from pending.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
peer_id: str,
|
||||
local_id: str,
|
||||
transport: DataTransport,
|
||||
local_block_len: int,
|
||||
conn: ControlConnection | None = None,
|
||||
) -> None:
|
||||
self.peer_id = peer_id
|
||||
self._local_id = local_id
|
||||
self._transport = transport
|
||||
self._local_block_len = local_block_len
|
||||
self._conn: ControlConnection | None = None
|
||||
|
||||
self._send_ready = False # True after the peer acked our ConnectMsg
|
||||
# Msgs waiting to be sent on connection establishment
|
||||
self._queued: list[dict] = []
|
||||
|
||||
# Consecutive non-protocol dispatch errors. Reset on success.
|
||||
self._dispatch_error_count: int = 0
|
||||
|
||||
# kv_request_ids whose FetchMsg arrived during the current poll
|
||||
# tick. Drained and returned in the next poll() result so the
|
||||
# manager can bind kv_request_id → session and replay any
|
||||
# submit_store batches parked before the binding existed.
|
||||
self._new_fetch_ids: list[str] = []
|
||||
|
||||
self._client = ClientRole(peer_id=peer_id, send=self._send)
|
||||
self._server = ServerRole(peer_id=peer_id, transport=transport, send=self._send)
|
||||
|
||||
if conn is not None:
|
||||
self.attach_connection(conn)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Properties
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
# Pending sessions (awaiting connection) are alive — only a
|
||||
# closed real connection counts as dead.
|
||||
return self._conn is None or self._conn.alive
|
||||
|
||||
@property
|
||||
def connected(self) -> bool:
|
||||
return self._conn is not None
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
"""True after the peer acked our ConnectMsg (we may send freely)."""
|
||||
return self._send_ready
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Connection lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def attach_connection(self, conn: ControlConnection) -> None:
|
||||
"""Attach a connection to a pending session and announce ourselves.
|
||||
|
||||
Symmetric: every side advertises its NIXL metadata on connect, so
|
||||
whichever peer receives a session first can register the other.
|
||||
"""
|
||||
if self._conn is not None:
|
||||
raise ValueError(f"P2PSession {self.peer_id}: already connected")
|
||||
self._conn = conn
|
||||
self._send_connect()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def request_blocks(
|
||||
self,
|
||||
job_id: JobId,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[bytes],
|
||||
block_ids: Sequence[int],
|
||||
) -> None:
|
||||
"""Send fetch to the peer."""
|
||||
self._client.request_blocks(
|
||||
job_id, kv_request_id, keys, block_ids, send_ready=self._send_ready
|
||||
)
|
||||
|
||||
def add_stored_blocks(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[OffloadKey],
|
||||
block_ids: Sequence[int],
|
||||
job_id: JobId,
|
||||
) -> None:
|
||||
"""New blocks stored locally — match against pending fetch demand."""
|
||||
self._server.add_stored_blocks(kv_request_id, keys, block_ids, job_id)
|
||||
|
||||
def finish_request(self, kv_request_id: str) -> None:
|
||||
"""Called when the request is finishing locally.
|
||||
|
||||
Cancels any inbound load (client role) and finalizes any
|
||||
outbound serving (server role) for this id. Roles that aren't
|
||||
active for this id are silent no-ops.
|
||||
"""
|
||||
self._client.cancel(kv_request_id)
|
||||
self._server.finish(kv_request_id)
|
||||
|
||||
def poll(self) -> SessionPollResult:
|
||||
"""Process incoming messages, drive transfers, apply timeouts."""
|
||||
if self._conn is None:
|
||||
# Pending session — store-job timeouts still apply so buffered
|
||||
# jobs that never get picked up are surfaced as failures.
|
||||
return SessionPollResult(
|
||||
loads=[],
|
||||
stores=self._server.collect_idle_timeouts(),
|
||||
new_fetch_ids=[],
|
||||
)
|
||||
|
||||
for msg in self._conn.recv():
|
||||
self._on_message(msg)
|
||||
|
||||
loads = self._client.collect_results()
|
||||
stores = self._server.collect_results()
|
||||
self._server.drain_pending_aborts()
|
||||
|
||||
new_fetch_ids = self._new_fetch_ids
|
||||
self._new_fetch_ids = []
|
||||
return SessionPollResult(
|
||||
loads=loads, stores=stores, new_fetch_ids=new_fetch_ids
|
||||
)
|
||||
|
||||
def close(self) -> tuple[list[tuple[int, str]], list[int]]:
|
||||
"""Shut down. Returns (failed_loads, failed_stores).
|
||||
|
||||
failed_loads: list of (job_id, kv_request_id) pairs.
|
||||
failed_stores: list of job_ids.
|
||||
"""
|
||||
failed_loads = self._client.close()
|
||||
failed_stores = self._server.close()
|
||||
|
||||
if self._conn is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
return failed_loads, failed_stores
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Message dispatch
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _on_message(self, msg: dict) -> None:
|
||||
msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else msg
|
||||
try:
|
||||
self._dispatch_message(msg)
|
||||
except ValueError as exc:
|
||||
# Protocol contract violation from the peer — *Msg.validate()
|
||||
# and handler-level checks raise ValueError. Retrying won't
|
||||
# help and may corrupt session state, so disconnect now.
|
||||
self._protocol_error(f"malformed {msg_type!r}: {exc}")
|
||||
return
|
||||
except Exception as exc:
|
||||
# Anything else is most likely an internal bug rather than a
|
||||
# peer fault. Log loudly with a traceback so it doesn't
|
||||
# disappear, but don't kill the session on a single hiccup.
|
||||
# Disconnect only if errors keep arriving — that pattern is
|
||||
# consistent with a peer wedging us into a broken state.
|
||||
self._dispatch_error_count += 1
|
||||
logger.exception(
|
||||
"P2PSession %s: error handling message %r (count=%d): %s",
|
||||
self.peer_id,
|
||||
msg_type,
|
||||
self._dispatch_error_count,
|
||||
exc,
|
||||
)
|
||||
if self._dispatch_error_count >= _MAX_CONSECUTIVE_DISPATCH_ERRORS:
|
||||
self._protocol_error(
|
||||
f"too many consecutive dispatch errors "
|
||||
f"({self._dispatch_error_count})"
|
||||
)
|
||||
return
|
||||
self._dispatch_error_count = 0
|
||||
|
||||
def _protocol_error(self, reason: str) -> None:
|
||||
"""Log a protocol violation and disconnect.
|
||||
|
||||
Best-effort sends ``DisconnectMsg`` so the peer learns why we're
|
||||
going away, then marks the connection dead. The manager reaps
|
||||
the session on the next poll via ``alive``.
|
||||
"""
|
||||
logger.error(
|
||||
"P2PSession %s: protocol error: %s — disconnecting",
|
||||
self.peer_id,
|
||||
reason,
|
||||
)
|
||||
if self._conn is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
|
||||
self._conn.mark_dead()
|
||||
|
||||
def _dispatch_message(self, msg: dict) -> None:
|
||||
# Drop messages buffered before disconnect: a poll batch can
|
||||
# contain msg-after-DisconnectMsg, and dispatching them would
|
||||
# mutate state on a dead session.
|
||||
if self._conn is not None and not self._conn.alive:
|
||||
return
|
||||
msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else None
|
||||
if msg_type == ConnectMsg.TYPE:
|
||||
self._on_connect(msg)
|
||||
elif msg_type == ConnectAckMsg.TYPE:
|
||||
ConnectAckMsg.validate(msg)
|
||||
self._on_connect_ack()
|
||||
elif msg_type == FetchMsg.TYPE:
|
||||
FetchMsg.validate(msg)
|
||||
kv_request_id = msg[FetchMsg.KV_REQUEST_ID]
|
||||
block_hashes = [
|
||||
OffloadKey(bh if isinstance(bh, bytes) else bytes(bh))
|
||||
for bh in msg[FetchMsg.BLOCK_HASHES]
|
||||
]
|
||||
block_indexes = msg[FetchMsg.BLOCK_INDEXES]
|
||||
# Run the server-role state machine inline as today —
|
||||
# add_fetch_demand records demand against any blocks we've
|
||||
# already seen in `available`. Report the kv_request_id so
|
||||
# the manager (after poll() returns) can replay any parked
|
||||
# submit_store batches; their add_stored_blocks calls hit
|
||||
# the demand recorded here and submit transfers immediately.
|
||||
self._server.on_fetch(kv_request_id, block_hashes, block_indexes)
|
||||
self._new_fetch_ids.append(kv_request_id)
|
||||
elif msg_type == AbortFetchMsg.TYPE:
|
||||
AbortFetchMsg.validate(msg)
|
||||
self._server.on_abort_fetch(msg[AbortFetchMsg.KV_REQUEST_ID])
|
||||
elif msg_type == TransferDoneMsg.TYPE:
|
||||
TransferDoneMsg.validate(msg)
|
||||
self._client.on_transfer_done(
|
||||
msg[TransferDoneMsg.KV_REQUEST_ID],
|
||||
msg[TransferDoneMsg.SUCCESS],
|
||||
)
|
||||
elif msg_type == AbortAckMsg.TYPE:
|
||||
AbortAckMsg.validate(msg)
|
||||
self._client.on_abort_ack(msg[AbortAckMsg.KV_REQUEST_ID])
|
||||
elif msg_type == DisconnectMsg.TYPE:
|
||||
if self._conn is not None:
|
||||
self._conn.mark_dead()
|
||||
else:
|
||||
logger.warning(
|
||||
"P2PSession %s: unknown message type %r", self.peer_id, msg_type
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Handshake
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _on_connect(self, msg: dict) -> None:
|
||||
# Validation failures here mean an incompatible or malicious peer.
|
||||
# Mark the connection dead so the manager reaps the session;
|
||||
# don't call add_remote_peer or send connect_ack.
|
||||
if self._send_ready:
|
||||
# We've already received connect_ack, so the handshake is
|
||||
# complete. A second connect from the peer is a protocol
|
||||
# violation — re-registering would corrupt transport state.
|
||||
self._protocol_error("duplicate connect after handshake")
|
||||
return
|
||||
try:
|
||||
ConnectMsg.validate(msg)
|
||||
if msg[ConnectMsg.BLOCK_LEN] != self._local_block_len:
|
||||
raise ValueError(
|
||||
f"block_len mismatch from {self.peer_id}: "
|
||||
f"remote={msg[ConnectMsg.BLOCK_LEN]}, "
|
||||
f"local={self._local_block_len}"
|
||||
)
|
||||
remote_fp = msg.get(ConnectMsg.CONFIG_FINGERPRINT, "")
|
||||
local_fp = self._transport.config_fingerprint
|
||||
if local_fp and remote_fp and remote_fp != local_fp:
|
||||
raise ValueError(
|
||||
f"config fingerprint mismatch from {self.peer_id}: "
|
||||
f"remote={remote_fp!r}, local={local_fp!r}"
|
||||
)
|
||||
self._transport.add_remote_peer(
|
||||
self.peer_id,
|
||||
agent_metadata=msg[ConnectMsg.AGENT_METADATA],
|
||||
base_addr=msg[ConnectMsg.BASE_ADDR],
|
||||
num_blocks=msg[ConnectMsg.NUM_BLOCKS],
|
||||
block_len=msg[ConnectMsg.BLOCK_LEN],
|
||||
)
|
||||
except ValueError as exc:
|
||||
logger.error("P2PSession %s: rejecting peer connect: %s", self.peer_id, exc)
|
||||
if self._conn is not None:
|
||||
self._conn.mark_dead()
|
||||
return
|
||||
|
||||
if self._conn is not None:
|
||||
self._conn.send(
|
||||
{
|
||||
TYPE_KEY: ConnectAckMsg.TYPE,
|
||||
ConnectAckMsg.PEER_ID: self._local_id,
|
||||
}
|
||||
)
|
||||
|
||||
def _on_connect_ack(self) -> None:
|
||||
if self._queued:
|
||||
logger.debug(
|
||||
"P2PSession %s: connect_ack received, flushing %d queued msg(s)",
|
||||
self.peer_id,
|
||||
len(self._queued),
|
||||
)
|
||||
self._send_ready = True
|
||||
for queued in self._queued:
|
||||
self._do_send(queued)
|
||||
self._queued.clear()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Send helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _send_connect(self) -> None:
|
||||
"""Send our ConnectMsg announcing local NIXL metadata."""
|
||||
assert self._conn is not None
|
||||
self._conn.send(
|
||||
{
|
||||
TYPE_KEY: ConnectMsg.TYPE,
|
||||
ConnectMsg.PEER_ID: self._local_id,
|
||||
ConnectMsg.AGENT_METADATA: self._transport.get_agent_metadata(),
|
||||
ConnectMsg.BASE_ADDR: self._transport.base_addr,
|
||||
ConnectMsg.NUM_BLOCKS: self._transport.num_blocks,
|
||||
ConnectMsg.BLOCK_LEN: self._transport.block_len,
|
||||
ConnectMsg.CONFIG_FINGERPRINT: self._transport.config_fingerprint,
|
||||
}
|
||||
)
|
||||
|
||||
def _send(self, msg: dict) -> None:
|
||||
if self._conn is None or not self._send_ready:
|
||||
logger.debug(
|
||||
"P2PSession %s: queueing %s (ready=%s queue_depth=%d)",
|
||||
self.peer_id,
|
||||
msg.get(TYPE_KEY),
|
||||
self._send_ready,
|
||||
len(self._queued) + 1,
|
||||
)
|
||||
self._queued.append(msg)
|
||||
return
|
||||
self._do_send(msg)
|
||||
|
||||
def _do_send(self, msg: dict) -> None:
|
||||
if self._conn is None:
|
||||
return
|
||||
try:
|
||||
self._conn.send(msg)
|
||||
logger.debug("P2PSession %s: sent %s", self.peer_id, msg.get(TYPE_KEY))
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"P2PSession %s: failed to send %s",
|
||||
self.peer_id,
|
||||
msg.get(TYPE_KEY),
|
||||
)
|
||||
Reference in New Issue
Block a user