Files
Liqian LinCodexRobert ShawItay Etelismergify[bot] <37929162+mergify[bot]@users.noreply.github.com>Or OzeriKevin H. Luu
3f1bcfd82b [Bugfix][KV Offload] Bound unanswered P2P lookup waits (#56836)
Signed-off-by: jacklin78911-collab <jacklin78911@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
2026-10-15 08:23:34 +03:00

667 lines
27 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""P2PSession — bidirectional session combining client + server roles.
A single P2PSession per remote peer handles BOTH directions of the P2P
protocol on one ControlConnection: it can request blocks from the peer
("client" role, in :mod:`.client`) AND serve blocks to the peer
("server" role, in :mod:`.server`). This module is the thin coordinator
that owns the connection, the handshake, send-gating, and the message
dispatch — each parsed message is forwarded to the corresponding role.
Wire protocol is unchanged. Both sides advertise their NIXL metadata
via ConnectMsg when their session is connected; the peer's ConnectMsg
starts transport.add_remote_peer_async off the scheduler thread, and
ConnectAckMsg is sent only once that registration lands. ConnectAckMsg
confirms the peer received our ConnectMsg, after which queued outgoing
messages are flushed — so a peer that may send is a peer we have already
registered.
"""
from __future__ import annotations
import contextlib
import time
from collections.abc import Sequence
from concurrent.futures import Future
from typing import TYPE_CHECKING, NamedTuple
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey
from vllm.v1.kv_offload.tiering.p2p.control.base import ControlConnection
from vllm.v1.kv_offload.tiering.p2p.session.client import ClientRole, LoadResult
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
TYPE_KEY,
AbortAckMsg,
AbortFetchMsg,
ConnectAckMsg,
ConnectMsg,
DisconnectMsg,
FetchMsg,
LookupMsg,
LookupRespMsg,
TransferDoneMsg,
)
from vllm.v1.kv_offload.tiering.p2p.session.server import (
ServerRole,
StoreResult,
)
if TYPE_CHECKING:
from vllm.v1.kv_offload.base import ReqContext
from vllm.v1.kv_offload.tiering.base import JobId, ParentManager
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport
logger = init_logger(__name__)
# Cap on consecutive non-protocol dispatch exceptions before we tear
# down the session. Protocol violations (ValueError) disconnect on the
# first occurrence; this threshold protects against repeated internal
# bugs that may indicate a peer-induced bad state. Reset on any
# successful dispatch.
_MAX_CONSECUTIVE_DISPATCH_ERRORS = 5
class SessionPollResult(NamedTuple):
"""Result of one P2PSession.poll() tick.
`loads`/`stores` are the same per-role results the manager has always
consumed. `new_fetch_ids` reports kv_request_ids whose FetchMsg
arrived this tick — the manager uses them to bind kv_request_id →
session and replay any submit_store batches parked while no peer had
asked yet. Reporting (rather than calling back into the manager
mid-dispatch) keeps the dependency strictly top-down.
"""
loads: list[LoadResult]
stores: list[StoreResult]
new_fetch_ids: list[str]
class SessionCloseResult(NamedTuple):
"""Result of tearing down a P2PSession.
`failed_jobs`/`failed_stores` are the in-flight jobs (client loads /
server stores) the manager must mark failed. `failed_req_ids` is every
client-side kv_request_id whose lookup() must fail (in-flight loads plus
unresolved probes); `failed_serves` is the server-side lookup state the
dead peer can no longer resolve.
"""
failed_jobs: list[int] # client load job_ids
failed_req_ids: list[str] # client kv_request_ids (loads + probes)
failed_stores: list[int] # server store job_ids
failed_serves: list[ReqContext] # server-side lookup ctxs needing release
class P2PSession:
"""Bidirectional session — coordinator over ClientRole + ServerRole.
Lifecycle:
- Constructor with conn=None ⇒ pending. Accepts add_stored_blocks
but cannot send (used by the prefiller to buffer blocks before
the decoder connects).
- Constructor with conn != None ⇒ connected. Sends our own ConnectMsg
immediately; the peer's ConnectMsg arrives in poll() and is
dispatched to _on_connect, which starts
transport.add_remote_peer_async. A later poll() sees the
registration land and replies with ConnectAckMsg; a failed
registration marks the connection dead instead. Outgoing sends are
queued until ConnectAckMsg confirms our metadata reached the peer.
- attach_connection(conn) on a pending session ⇒ same as above,
starting from pending.
"""
def __init__(
self,
peer_id: str,
local_id: str,
transport: DataTransport,
local_block_len: int,
local_hash_seed: str,
conn: ControlConnection | None = None,
) -> None:
self.peer_id = peer_id
self._local_id = local_id
self._transport = transport
self._local_block_len = local_block_len
self._local_hash_seed = local_hash_seed
self._conn: ControlConnection | None = None
self._send_ready = False # True after the peer acked our ConnectMsg
# Msgs waiting to be sent on connection establishment
self._queued: list[dict] = []
# Consecutive non-protocol dispatch errors. Reset on success.
self._dispatch_error_count: int = 0
now = time.monotonic()
self._connected_at: float | None = None
self._last_activity_at: float = now
# kv_request_ids whose FetchMsg arrived during the current poll
# tick. Drained and returned in the next poll() result so the
# manager can bind kv_request_id → session and replay any
# submit_store batches parked before the binding existed.
self._new_fetch_ids: list[str] = []
# In-flight peer registration started by _on_connect. Registration
# is O(num_blocks) and runs off this thread, so ConnectAckMsg is
# held back until the future resolves — the ack is what tells the
# peer it may start sending, and nothing may be served before the
# peer's blocks are registered here.
self._pending_registration: Future[None] | None = None
self._client = ClientRole(peer_id=peer_id, send=self._send)
self._server = ServerRole(
peer_id=peer_id,
transport=transport,
send=self._send,
)
if conn is not None:
self.attach_connection(conn)
# ------------------------------------------------------------------
# Properties
# ------------------------------------------------------------------
@property
def alive(self) -> bool:
# Pending sessions (awaiting connection) are alive — only a
# closed real connection counts as dead.
return self._conn is None or self._conn.alive
@property
def connected(self) -> bool:
return self._conn is not None
@property
def ready(self) -> bool:
"""True after the peer acked our ConnectMsg (we may send freely)."""
return self._send_ready
@property
def has_pending_work(self) -> bool:
"""True while inbound loads or outbound transfers are outstanding."""
return (
self._pending_registration is not None
or self._client.has_active_loads
or self._server.has_inflight_transfers
)
@property
def last_activity_at(self) -> float:
"""Monotonic timestamp of the last meaningful session activity."""
return self._last_activity_at
# ------------------------------------------------------------------
# Connection lifecycle
# ------------------------------------------------------------------
def attach_connection(self, conn: ControlConnection) -> None:
"""Attach a connection to a pending session and announce ourselves.
Symmetric: every side advertises its NIXL metadata on connect, so
whichever peer receives a session first can register the other.
"""
if self._conn is not None:
raise ValueError(f"P2PSession {self.peer_id}: already connected")
self._conn = conn
now = time.monotonic()
self._connected_at = now
self._last_activity_at = now
self._send_connect()
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def request_blocks(
self,
job_id: JobId,
kv_request_id: str,
keys: Sequence[OffloadKey],
block_ids: Sequence[int],
) -> None:
"""Send fetch to the peer."""
self._client.request_blocks(
job_id, kv_request_id, keys, block_ids, send_ready=self._send_ready
)
def add_stored_blocks(
self,
kv_request_id: str,
keys: Sequence[OffloadKey],
block_ids: Sequence[int],
job_id: JobId,
) -> None:
"""New blocks stored locally — match against pending fetch demand."""
self._server.add_stored_blocks(kv_request_id, keys, block_ids, job_id)
def finish_request(self, kv_request_id: str) -> None:
"""Called when the request is finishing locally.
Finishes the client role (aborts any inbound load and drops any
pending symmetric-P2P lookup state) and finalizes any outbound
serving (server role) for this id. Roles that aren't active for
this id are silent no-ops.
"""
self._client.finish(kv_request_id)
self._server.finish(kv_request_id)
def register_lookup(self, kv_request_id: str, key: bytes) -> bool | None:
"""Register or resolve one (kv_request_id, key) probe.
Called from the manager's lookup() for symmetric-P2P consumers
(``remote_kv_source`` sub-dict in kv_transfer_params). See
``ClientRole.register_lookup`` for the state-machine contract.
"""
return self._client.register_lookup(kv_request_id, key)
def flush_pending_lookups(self) -> None:
"""Flush any aggregated symmetric-P2P lookups for this peer.
Called once per scheduler step from the manager's
``on_schedule_end()``. Probes wait for ConnectAckMsg before sending.
"""
self._client.flush_pending_lookups(send_ready=self._send_ready)
def serve_external_requests(self, parent: ParentManager) -> None:
"""Resolve inbound peer lookups against the tiering manager.
Delegates to the server role; the ``parent`` handle is valid
only for the duration of this call.
"""
self._server.serve_external_requests(parent)
def poll(self) -> SessionPollResult:
"""Process incoming messages, drive transfers, apply timeouts."""
if self._conn is None:
return SessionPollResult(
loads=[],
stores=self._server.collect_idle_timeouts(),
new_fetch_ids=[],
)
# Drain the inbox first so a ConnectAck already buffered at the
# deadline (e.g. after a scheduler stall) can complete the
# handshake instead of being discarded by mark_dead.
messages = self._conn.recv()
if messages:
self._last_activity_at = time.monotonic()
for msg in messages:
self._on_message(msg)
# After dispatch: a ConnectMsg handled just above may already have a
# resolved registration (transports that register inline ack in the
# same tick), and a registration started on an earlier tick may have
# landed while the engine was busy elsewhere. This runs before the
# handshake deadline below so a registration that just resolved gets
# its ConnectAckMsg out rather than being judged as timed out, and so
# a failed registration's mark_dead() short-circuits that check.
self._poll_registration()
# Handshake deadline: a peer that never completes the handshake
# is stuck in connected-but-not-ready state. Mark it dead so the
# manager reaps it instead of accumulating sockets indefinitely.
# Note this also covers the peer registering *our* region before it
# acks, so the deadline must exceed remote registration time; the
# 30s default leaves ample room for the descriptor build.
if (
not self._send_ready
and self._conn.alive
and self._connected_at is not None
and time.monotonic() - self._connected_at
> envs.VLLM_P2P_HANDSHAKE_TIMEOUT_S
):
logger.warning(
"P2PSession %s: handshake timed out after %ds "
"— marking connection dead",
self.peer_id,
envs.VLLM_P2P_HANDSHAKE_TIMEOUT_S,
)
self._conn.mark_dead()
loads = self._client.collect_results()
stores = self._server.collect_results()
self._server.drain_pending_aborts()
# Drain buffered replies and completed transfers before expiring a peer.
expired_lookup = self._client.get_expired_lookup()
if expired_lookup is not None:
req_id, elapsed = expired_lookup
logger.warning(
"P2PSession %s: lookup response timed out for kv_request_id=%s "
"after %.1fs without progress; disconnecting peer session",
self.peer_id,
req_id,
elapsed,
)
with contextlib.suppress(Exception):
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
self._conn.mark_dead()
new_fetch_ids = self._new_fetch_ids
self._new_fetch_ids = []
return SessionPollResult(
loads=loads, stores=stores, new_fetch_ids=new_fetch_ids
)
def close(self) -> SessionCloseResult:
"""Shut down.
failed_jobs: client load job_ids to fail.
failed_req_ids: client kv_request_ids to fail — in-flight loads plus
requests with an unresolved symmetric-P2P probe toward the
now-dead peer. The manager fails these so the consumer's lookup()
falls back to local prefill instead of deferring forever on an
answer that can never arrive.
failed_stores: server store job_ids to fail.
failed_serves: synthetic lookup ctxs still owing
``parent.on_request_finished`` (the manager flushes these on
its next ``serve_external_requests``).
"""
# Abandon any in-flight registration: nothing will ack this peer
# now. The manager's remove_remote_peer bumps the transport's
# generation for this peer, so a worker already past cancellation
# releases its handles instead of publishing them.
if self._pending_registration is not None:
self._pending_registration.cancel()
self._pending_registration = None
client_result = self._client.close()
failed_stores, failed_serves = self._server.close()
if self._conn is not None:
with contextlib.suppress(Exception):
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
self._conn.close()
self._conn = None
return SessionCloseResult(
failed_jobs=client_result.failed_jobs,
failed_req_ids=client_result.failed_req_ids,
failed_stores=failed_stores,
failed_serves=failed_serves,
)
# ------------------------------------------------------------------
# Message dispatch
# ------------------------------------------------------------------
def _on_message(self, msg: dict) -> None:
msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else msg
try:
self._dispatch_message(msg)
except ValueError as exc:
# Protocol contract violation from the peer — *Msg.validate()
# and handler-level checks raise ValueError. Retrying won't
# help and may corrupt session state, so disconnect now.
self._protocol_error(f"malformed {msg_type!r}: {exc}")
return
except Exception as exc:
# Anything else is most likely an internal bug rather than a
# peer fault. Log loudly with a traceback so it doesn't
# disappear, but don't kill the session on a single hiccup.
# Disconnect only if errors keep arriving — that pattern is
# consistent with a peer wedging us into a broken state.
self._dispatch_error_count += 1
logger.exception(
"P2PSession %s: error handling message %r (count=%d): %s",
self.peer_id,
msg_type,
self._dispatch_error_count,
exc,
)
if self._dispatch_error_count >= _MAX_CONSECUTIVE_DISPATCH_ERRORS:
self._protocol_error(
f"too many consecutive dispatch errors "
f"({self._dispatch_error_count})"
)
return
self._dispatch_error_count = 0
def _protocol_error(self, reason: str) -> None:
"""Log a protocol violation and disconnect.
Best-effort sends ``DisconnectMsg`` so the peer learns why we're
going away, then marks the connection dead. The manager reaps
the session on the next poll via ``alive``.
"""
logger.error(
"P2PSession %s: protocol error: %s — disconnecting",
self.peer_id,
reason,
)
if self._conn is not None:
with contextlib.suppress(Exception):
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
self._conn.mark_dead()
def _dispatch_message(self, msg: dict) -> None:
# Drop messages buffered before disconnect: a poll batch can
# contain msg-after-DisconnectMsg, and dispatching them would
# mutate state on a dead session.
if self._conn is not None and not self._conn.alive:
return
msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else None
if msg_type == ConnectMsg.TYPE:
self._on_connect(msg)
elif msg_type == ConnectAckMsg.TYPE:
ConnectAckMsg.validate(msg)
self._on_connect_ack()
elif msg_type == FetchMsg.TYPE:
FetchMsg.validate(msg)
kv_request_id = msg[FetchMsg.KV_REQUEST_ID]
keys = [
OffloadKey(bh if isinstance(bh, bytes) else bytes(bh))
for bh in msg[FetchMsg.KEYS]
]
block_indexes = msg[FetchMsg.BLOCK_INDEXES]
round_seq = msg[FetchMsg.ROUND_SEQ]
# Run the server-role state machine inline as today —
# add_fetch_demand records demand against any blocks we've
# already seen in `available`. Report the kv_request_id so
# the manager (after poll() returns) can replay any parked
# submit_store batches; their add_stored_blocks calls hit
# the demand recorded here and submit transfers immediately.
self._server.on_fetch(kv_request_id, keys, block_indexes, round_seq)
self._new_fetch_ids.append(kv_request_id)
elif msg_type == AbortFetchMsg.TYPE:
AbortFetchMsg.validate(msg)
self._server.on_abort_fetch(
msg[AbortFetchMsg.KV_REQUEST_ID],
msg[AbortFetchMsg.ROUND_SEQ],
)
elif msg_type == TransferDoneMsg.TYPE:
TransferDoneMsg.validate(msg)
self._client.on_transfer_done(
msg[TransferDoneMsg.KV_REQUEST_ID],
msg[TransferDoneMsg.SUCCESS],
msg[TransferDoneMsg.ROUND_SEQ],
)
elif msg_type == AbortAckMsg.TYPE:
AbortAckMsg.validate(msg)
self._client.on_abort_ack(
msg[AbortAckMsg.KV_REQUEST_ID],
msg[AbortAckMsg.ROUND_SEQ],
)
elif msg_type == LookupMsg.TYPE:
LookupMsg.validate(msg)
kv_request_id = msg[LookupMsg.KV_REQUEST_ID]
keys = [
OffloadKey(bh if isinstance(bh, bytes) else bytes(bh))
for bh in msg[LookupMsg.KEYS]
]
self._server.on_lookup(kv_request_id, keys, msg[LookupMsg.ROUND_SEQ])
elif msg_type == LookupRespMsg.TYPE:
LookupRespMsg.validate(msg)
kv_request_id = msg[LookupRespMsg.KV_REQUEST_ID]
keys = [
OffloadKey(bh if isinstance(bh, bytes) else bytes(bh))
for bh in msg[LookupRespMsg.KEYS]
]
hits = msg[LookupRespMsg.HITS]
self._client.on_lookup_resp(kv_request_id, keys, hits)
elif msg_type == DisconnectMsg.TYPE:
if self._conn is not None:
self._conn.mark_dead()
else:
logger.warning(
"P2PSession %s: unknown message type %r", self.peer_id, msg_type
)
# ------------------------------------------------------------------
# Handshake
# ------------------------------------------------------------------
def _on_connect(self, msg: dict) -> None:
# Validation failures here mean an incompatible or malicious peer.
# Mark the connection dead so the manager reaps the session;
# don't call add_remote_peer or send connect_ack.
if self._send_ready or self._pending_registration is not None:
# Either the handshake completed (we hold the peer's ack) or a
# registration for this peer is already in flight. A second
# connect either way is a protocol violation — re-registering
# would corrupt transport state.
self._protocol_error("duplicate connect after handshake")
return
try:
ConnectMsg.validate(msg)
if msg[ConnectMsg.BLOCK_LEN] != self._local_block_len:
raise ValueError(
f"block_len mismatch from {self.peer_id}: "
f"remote={msg[ConnectMsg.BLOCK_LEN]}, "
f"local={self._local_block_len}"
)
remote_fp = msg.get(ConnectMsg.CONFIG_FINGERPRINT, "")
local_fp = self._transport.config_fingerprint
if local_fp and remote_fp and remote_fp != local_fp:
raise ValueError(
f"config fingerprint mismatch from {self.peer_id}: "
f"remote={remote_fp!r}, local={local_fp!r}"
)
if msg[ConnectMsg.HASH_SEED] != self._local_hash_seed:
raise ValueError(
f"hash seed mismatch from {self.peer_id}: "
f"remote={msg[ConnectMsg.HASH_SEED]!r}, "
f"local={self._local_hash_seed!r}. Ensure PYTHONHASHSEED "
"(if set) matches on all P2P peers."
)
registration = self._transport.add_remote_peer_async(
self.peer_id,
agent_metadata=msg[ConnectMsg.AGENT_METADATA],
base_addr=msg[ConnectMsg.BASE_ADDR],
num_blocks=msg[ConnectMsg.NUM_BLOCKS],
block_len=msg[ConnectMsg.BLOCK_LEN],
)
except ValueError as exc:
logger.error("P2PSession %s: rejecting peer connect: %s", self.peer_id, exc)
if self._conn is not None:
self._conn.mark_dead()
return
# ConnectAckMsg is deferred to _poll_registration once the
# registration lands; sending it now would let the peer fetch
# against blocks this side has not registered yet.
self._pending_registration = registration
def _poll_registration(self) -> None:
"""Ack or reject the peer once its registration resolves.
Called from poll() after message dispatch. A successful
registration releases ConnectAckMsg — the peer treats that as
permission to send, so it must not go out earlier. A failed one
marks the connection dead so the manager reaps the session.
"""
registration = self._pending_registration
if registration is None or not registration.done():
return
self._pending_registration = None
try:
registration.result()
except Exception as exc:
logger.error(
"P2PSession %s: peer registration failed, rejecting peer: %s",
self.peer_id,
exc,
)
if self._conn is not None:
self._conn.mark_dead()
return
if self._conn is not None and self._conn.alive:
self._conn.send(
{
TYPE_KEY: ConnectAckMsg.TYPE,
ConnectAckMsg.PEER_ID: self._local_id,
}
)
def _on_connect_ack(self) -> None:
if self._queued:
logger.debug(
"P2PSession %s: connect_ack received, flushing %d queued msg(s)",
self.peer_id,
len(self._queued),
)
self._send_ready = True
for queued in self._queued:
self._do_send(queued)
self._queued.clear()
# ------------------------------------------------------------------
# Send helpers
# ------------------------------------------------------------------
def _send_connect(self) -> None:
"""Send our ConnectMsg announcing local NIXL metadata."""
assert self._conn is not None
self._conn.send(
{
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: self._local_id,
ConnectMsg.AGENT_METADATA: self._transport.get_agent_metadata(),
ConnectMsg.BASE_ADDR: self._transport.base_addr,
ConnectMsg.NUM_BLOCKS: self._transport.num_blocks,
ConnectMsg.BLOCK_LEN: self._transport.block_len,
ConnectMsg.CONFIG_FINGERPRINT: self._transport.config_fingerprint,
ConnectMsg.HASH_SEED: self._local_hash_seed,
}
)
def _send(self, msg: dict) -> None:
if self._conn is None or not self._send_ready:
logger.debug(
"P2PSession %s: queueing %s (ready=%s queue_depth=%d)",
self.peer_id,
msg.get(TYPE_KEY),
self._send_ready,
len(self._queued) + 1,
)
self._queued.append(msg)
return
self._do_send(msg)
def _do_send(self, msg: dict) -> None:
if self._conn is None:
return
try:
self._conn.send(msg)
self._last_activity_at = time.monotonic()
logger.debug("P2PSession %s: sent %s", self.peer_id, msg.get(TYPE_KEY))
except Exception:
# A send failure means the connection is broken. Swallowing it
# silently strands every in-flight lookup/load toward this peer:
# the session stays alive, is never reaped, and the consumer's
# lookup() keeps returning RETRY until the HTTP client times out.
# Mark the connection dead so the manager reaps the session on
# its next poll and surfaces the stranded work as failures.
logger.warning(
"P2PSession %s: send of %s failed — marking connection dead",
self.peer_id,
msg.get(TYPE_KEY),
)
self._conn.mark_dead()