Signed-off-by: Liran Schour <lirans@il.ibm.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
1282 lines
53 KiB
Python
1282 lines
53 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Server-role state machine for a single peer session.
|
|
|
|
Owns block matching (supply vs. demand), inflight RDMA transfers, store-job
|
|
timeouts, abort-drain, and produces ``StoreResult`` for completed stores.
|
|
The session coordinator parses wire messages and dispatches typed
|
|
arguments here; this module never touches ``ControlConnection`` directly
|
|
— it emits via the ``send`` callback injected by the coordinator (which
|
|
gates on ConnectAck).
|
|
|
|
Protocol violations the role can detect (today: duplicate ``FetchMsg``
|
|
for the same ``kv_request_id``) are surfaced as ``ValueError`` so the
|
|
coordinator's ``_dispatch_message`` can reuse its existing
|
|
``_protocol_error`` path.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from collections.abc import Callable, Iterable, Sequence
|
|
from dataclasses import dataclass, field
|
|
from typing import TYPE_CHECKING, NamedTuple
|
|
|
|
from vllm.logger import init_logger
|
|
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
|
|
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
|
|
TYPE_KEY,
|
|
AbortAckMsg,
|
|
LookupRespMsg,
|
|
TransferDoneMsg,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.v1.kv_offload.tiering.base import JobId, ParentManager
|
|
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
_STORE_TIMEOUT_S = 30.0
|
|
_CANCEL_DRAIN_TIMEOUT_S = 10.0
|
|
# Minimum spacing between wait-mode cancel attempts for one round. Each
|
|
# attempt calls into NIXL on the scheduler thread, so an unconfirmed
|
|
# cancellation is retried at this rate instead of on every poll tick.
|
|
_CANCEL_RETRY_INTERVAL_S = 0.1
|
|
# Cap on time the server holds a HIT_PENDING / RETRY key from an inbound
|
|
# LookupMsg before falling back to MISS. Long enough that an in-flight
|
|
# primary write or a just-started promotion typically completes; short
|
|
# enough that the consumer doesn't sit idle on a stuck producer.
|
|
_LOOKUP_PENDING_TIMEOUT_S = 5.0
|
|
|
|
|
|
class StoreResult(NamedTuple):
|
|
"""Result from a session poll, server side."""
|
|
|
|
job_id: int
|
|
success: bool
|
|
|
|
|
|
class _MatchResult(NamedTuple):
|
|
"""Result of block matching: pairs ready for transfer."""
|
|
|
|
local_idxs: list[int]
|
|
remote_idxs: list[int]
|
|
# The set of store job IDs that contributed blocks
|
|
job_ids: set[int]
|
|
|
|
|
|
@dataclass
|
|
class _OutboundRequestState:
|
|
"""Server-role state for a single fetch round of a peer request.
|
|
|
|
A kv_request_id may run several lookup→fetch rounds; rounds live in
|
|
``_ServerRequestState.outbound`` keyed by the wire ``round_seq``, so
|
|
terminals touch only their own round.
|
|
"""
|
|
|
|
# Supply came from inbound lookup pins (symmetric): no late
|
|
# submit_store can arrive, so unmatched fetch demand fails fast. PD
|
|
# rounds park demand for stores instead.
|
|
lookup_supplied: bool = False
|
|
demand_received: bool = False
|
|
available: dict[OffloadKey, tuple[int, int]] = field(
|
|
default_factory=dict
|
|
) # key → (job_id, local_block_idx): blocks we have, awaiting demand
|
|
demanded: dict[OffloadKey, int] = field(
|
|
default_factory=dict
|
|
) # key → remote_block_idx: blocks peer wants, awaiting supply
|
|
remaining: int = 0 # blocks that need to be transferred to client
|
|
finishing: bool = False # Signal finish request ASAP
|
|
inflight: int = 0 # transfers submitted for this round, not yet polled
|
|
# The round was cancelled (a store job timed out, or the peer aborted
|
|
# it) while transfers may still be writing to the peer. Its jobs fail,
|
|
# but a job keeps its pin until every transfer carrying it is terminal:
|
|
# NIXL cannot stop a WRITE that is already in flight.
|
|
cancelled: bool = False
|
|
# Last wait-mode cancel attempt for this round's transfers (monotonic).
|
|
last_cancel_at: float | None = None
|
|
# Job IDs that submit_store'd blocks for this round and have not
|
|
# yet emitted a StoreResult. The terminal-finalize helper drains
|
|
# this set; poll-done and poll-failed discard entries as their
|
|
# StoreResults fire.
|
|
pending_job_ids: set[int] = field(default_factory=set)
|
|
|
|
def add_stored_blocks(
|
|
self,
|
|
keys: Sequence[OffloadKey],
|
|
block_ids: Sequence[int],
|
|
job_id: int,
|
|
) -> _MatchResult:
|
|
"""Add locally-stored blocks. Returns matched pairs."""
|
|
self.pending_job_ids.add(job_id)
|
|
local_idxs: list[int] = []
|
|
remote_idxs: list[int] = []
|
|
for key, local_idx in zip(keys, block_ids):
|
|
remote_idx = self.demanded.pop(key, None)
|
|
if remote_idx is not None:
|
|
local_idxs.append(local_idx)
|
|
remote_idxs.append(remote_idx)
|
|
else:
|
|
self.available[key] = (job_id, local_idx)
|
|
return _MatchResult(
|
|
local_idxs=local_idxs,
|
|
remote_idxs=remote_idxs,
|
|
job_ids={job_id} if local_idxs else set(),
|
|
)
|
|
|
|
def add_fetch_demand(
|
|
self,
|
|
keys: Sequence[OffloadKey],
|
|
block_indexes: Sequence[int],
|
|
) -> _MatchResult:
|
|
"""Register the peer's fetch demand. Returns matched pairs."""
|
|
self.demand_received = True
|
|
self.remaining = len(keys)
|
|
|
|
local_idxs: list[int] = []
|
|
remote_idxs: list[int] = []
|
|
job_ids: set[int] = set()
|
|
for key, remote_idx in zip(keys, block_indexes):
|
|
stored_entry = self.available.pop(key, None)
|
|
if stored_entry is not None:
|
|
stored_job_id, local_idx = stored_entry
|
|
local_idxs.append(local_idx)
|
|
remote_idxs.append(remote_idx)
|
|
job_ids.add(stored_job_id)
|
|
else:
|
|
self.demanded[key] = remote_idx
|
|
return _MatchResult(
|
|
local_idxs=local_idxs,
|
|
remote_idxs=remote_idxs,
|
|
job_ids=job_ids,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class _InflightXfer:
|
|
"""Metadata for a single inflight RDMA transfer, keyed by transfer_id."""
|
|
|
|
kv_request_id: str
|
|
block_count: int
|
|
# The set of store job IDs that contributed blocks to this transfer.
|
|
job_ids: set[int]
|
|
# Round this transfer serves and its key in ``st.outbound``;
|
|
# remaining/finalize apply only while the round is still registered.
|
|
# Dummy default for test-seeded entries.
|
|
round: _OutboundRequestState = field(default_factory=_OutboundRequestState)
|
|
round_key: int = 0
|
|
|
|
|
|
@dataclass
|
|
class _ActiveLookup:
|
|
"""In-flight state for one inbound LookupMsg.
|
|
|
|
Aggregates per-key HIT/MISS resolutions and defers the single
|
|
outbound LookupRespMsg until every key has been resolved or the
|
|
``deadline`` fires (remaining ``pending`` keys then force-MISS).
|
|
Exactly one LookupRespMsg is emitted per LookupMsg, carrying every
|
|
key in the original wire order.
|
|
"""
|
|
|
|
lookup_id: int
|
|
kv_request_id: str
|
|
ctx: ReqContext
|
|
# Wire round these probes belong to; pins park under it.
|
|
round_seq: int = 0
|
|
# Keys from the inbound LookupMsg, preserved in wire order so
|
|
# the aggregated response goes back in the same order.
|
|
keys: list[OffloadKey] = field(default_factory=list)
|
|
# Per-key resolution: True = HIT, False = MISS. A key is present
|
|
# here once definitively resolved; still-pending keys are only
|
|
# in ``pending``.
|
|
resolved: dict[OffloadKey, bool] = field(default_factory=dict)
|
|
# Keys still awaiting resolution (HIT_PENDING / RETRY from
|
|
# ``parent.lookup``); re-polled by ``_resolve_pending_lookups``.
|
|
pending: set[OffloadKey] = field(default_factory=set)
|
|
# Absolute deadline (``time.monotonic``). Once reached, remaining
|
|
# ``pending`` keys are force-resolved to MISS so the consumer
|
|
# can fall back instead of waiting on a stuck producer.
|
|
deadline: float = 0.0
|
|
|
|
|
|
class _PendingLookup(NamedTuple):
|
|
"""A raw inbound LookupMsg awaiting resolution.
|
|
|
|
Enqueued by ``on_lookup`` during dispatch and drained by the next
|
|
``serve_external_requests``. The deadline for any resulting
|
|
HIT_PENDING / RETRY key is measured from ``enqueued_at``.
|
|
"""
|
|
|
|
keys: list[OffloadKey]
|
|
enqueued_at: float
|
|
round_seq: int = 0
|
|
|
|
|
|
@dataclass
|
|
class _ServerRequestState:
|
|
"""Per-kv_request_id server-side state.
|
|
|
|
Consolidates the outbound serve/transfer state, the symmetric-P2P
|
|
inbound-lookup state, abort bookkeeping, and the inflight-transfer
|
|
count for one kv_request_id. The owning id is the ``_requests`` dict
|
|
key and is not duplicated here. An entry is dropped once every field
|
|
is idle — see ``ServerRole._maybe_prune``.
|
|
"""
|
|
|
|
# Fetch rounds keyed by wire round_seq. A round is created by its
|
|
# first supply or its fetch and removed at its terminal (finalize /
|
|
# failure / abort).
|
|
outbound: dict[int, _OutboundRequestState] = field(default_factory=dict)
|
|
# Raw inbound LookupMsgs not yet processed against the ParentManager.
|
|
pending_lookups: list[_PendingLookup] = field(default_factory=list)
|
|
# Per-LookupMsg state parked with HIT_PENDING / RETRY keys, keyed by
|
|
# the (globally unique) lookup_id and re-polled each serve.
|
|
lookups: dict[int, _ActiveLookup] = field(default_factory=dict)
|
|
# Transfer ids in ``ServerRole._inflight`` for this id. Kept in sync
|
|
# via _inflight_add / _inflight_pop so a non-empty set is an exact
|
|
# "has any inflight transfer" predicate and the abort drain can
|
|
# enumerate this request's transfers without scanning all of
|
|
# ``_inflight``.
|
|
inflight_tids: set[int] = field(default_factory=set)
|
|
|
|
|
|
class ServerRole:
|
|
"""Server-side store/serve state machine for one peer session.
|
|
|
|
The coordinator owns the connection and the send-gating; this role
|
|
is given a ``send`` callback, the ``DataTransport``, and the
|
|
``peer_id`` for transport calls and log messages.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
peer_id: str,
|
|
transport: DataTransport,
|
|
send: Callable[[dict], None],
|
|
) -> None:
|
|
self._peer_id = peer_id
|
|
self._transport = transport
|
|
self._send = send
|
|
|
|
# All per-kv_request_id state lives here. Entries are created
|
|
# lazily and dropped by _maybe_prune once every field is idle.
|
|
self._requests: dict[str, _ServerRequestState] = {}
|
|
# kv_request_ids with lookup work (unprocessed pending_lookups or
|
|
# parked lookups) for the next serve to visit — the work-list that
|
|
# keeps serve_external_requests from scanning every request.
|
|
self._serve_pending: set[str] = set()
|
|
# transfer_id → xfer. Mutate ONLY via _inflight_add / _inflight_pop
|
|
# so the per-request inflight_tids stays in sync.
|
|
self._inflight: dict[int, _InflightXfer] = {}
|
|
self._store_jobs: dict[int, float] = {} # job_id → submitted_at
|
|
# StoreResults queued by _finalize_outbound for the next poll
|
|
# tick to surface. Mirrors the deferred-result pattern used for
|
|
# load timeouts.
|
|
self._pending_store_results: list[StoreResult] = []
|
|
# Synthetic lookup ctxs whose ``on_request_finished`` still needs to
|
|
# fire but which were closed outside a serve window (FetchMsg / local
|
|
# finish popped their parked lookup). Drained via
|
|
# ``parent.on_request_finished`` in ``serve_external_requests``.
|
|
self._finished_lookup_ctxs: list[ReqContext] = []
|
|
self._lookup_id_counter: int = 0
|
|
# Parked aborts awaiting drain, keyed by (kv_request_id, round)
|
|
# with the abort start time.
|
|
self._pending_aborts: dict[tuple[str, int], float] = {}
|
|
# Parked aborts already warned about for exceeding
|
|
# _CANCEL_DRAIN_TIMEOUT_S.
|
|
self._slow_aborts: set[tuple[str, int]] = set()
|
|
|
|
# ------------------------------------------------------------------
|
|
# State helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
def _get_or_create_request(self, kv_request_id: str) -> _ServerRequestState:
|
|
"""Get or create the state entry for a kv_request_id."""
|
|
st = self._requests.get(kv_request_id)
|
|
if st is None:
|
|
st = _ServerRequestState()
|
|
self._requests[kv_request_id] = st
|
|
return st
|
|
|
|
def _maybe_prune(self, kv_request_id: str) -> None:
|
|
"""Drop the entry once it holds no live state."""
|
|
st = self._requests.get(kv_request_id)
|
|
if (
|
|
st is not None
|
|
and not st.outbound
|
|
and not st.inflight_tids
|
|
and not st.lookups
|
|
and not st.pending_lookups
|
|
and not any(kv == kv_request_id for kv, _ in self._pending_aborts)
|
|
):
|
|
del self._requests[kv_request_id]
|
|
|
|
# ------------------------------------------------------------------
|
|
# Public API
|
|
# ------------------------------------------------------------------
|
|
|
|
def add_stored_blocks(
|
|
self,
|
|
kv_request_id: str,
|
|
keys: Sequence[OffloadKey],
|
|
block_ids: Sequence[int],
|
|
job_id: JobId,
|
|
round_seq: int = 0,
|
|
*,
|
|
from_lookup: bool = False,
|
|
) -> None:
|
|
"""New blocks stored locally — match within their fetch round.
|
|
|
|
Lookup pins carry the round they were probed under; PD
|
|
submit_store batches share PD's single round 0.
|
|
"""
|
|
self._store_jobs[job_id] = time.monotonic()
|
|
st = self._get_or_create_request(kv_request_id)
|
|
rnd = st.outbound.get(round_seq)
|
|
if rnd is None:
|
|
rnd = st.outbound[round_seq] = _OutboundRequestState()
|
|
if rnd.cancelled:
|
|
self._store_jobs.pop(job_id)
|
|
self._pending_store_results.append(StoreResult(job_id, success=False))
|
|
return
|
|
if from_lookup:
|
|
rnd.lookup_supplied = True
|
|
result = rnd.add_stored_blocks(keys, block_ids, job_id)
|
|
if result.local_idxs and rnd.demand_received:
|
|
self._submit_transfer(kv_request_id, result, rnd, round_seq)
|
|
|
|
def on_fetch(
|
|
self,
|
|
kv_request_id: str,
|
|
keys: Sequence[OffloadKey],
|
|
block_indexes: Sequence[int],
|
|
round_seq: int = 0,
|
|
) -> None:
|
|
"""Handle a FetchMsg from the peer.
|
|
|
|
A non-empty fetch binds and closes its round, leaving lookup
|
|
state alone (the next round's LookupMsg may already be in
|
|
flight). The terminal empty fetch closes the id: parked lookups
|
|
are popped and every remaining round drained. A second fetch for
|
|
a round already holding demand raises ValueError
|
|
(protocol-error disconnect).
|
|
"""
|
|
logger.debug(
|
|
"P2PSession %s: fetch RECEIVED kv_request_id=%s round=%s blocks=%d",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
round_seq,
|
|
len(keys),
|
|
)
|
|
st = self._requests.get(kv_request_id)
|
|
existing = st.outbound.get(round_seq) if st is not None else None
|
|
if existing is not None and existing.demand_received:
|
|
raise ValueError(
|
|
f"duplicate fetch for kv_request_id={kv_request_id} round={round_seq}"
|
|
)
|
|
st = self._get_or_create_request(kv_request_id)
|
|
req = st.outbound.get(round_seq)
|
|
if req is None:
|
|
req = st.outbound[round_seq] = _OutboundRequestState()
|
|
result = req.add_fetch_demand(keys, block_indexes)
|
|
if not keys:
|
|
# Terminal empty fetch: close the lookup phase and drain
|
|
# every round with no TransferDoneMsg (nothing waits on it).
|
|
self._finish_inbound_lookups(kv_request_id)
|
|
for key in list(st.outbound):
|
|
self._finalize_outbound(kv_request_id, key, send_done=False)
|
|
return
|
|
if req.lookup_supplied and req.demanded:
|
|
# A symmetric round's supply always precedes its fetch, so
|
|
# unmatched demand is unservable — fail now, not at the load
|
|
# timeout. PD rounds keep parking demand for stores that
|
|
# arrive later.
|
|
logger.warning(
|
|
"P2PSession %s: fetch kv_request_id=%s round=%s demanded %d "
|
|
"blocks but %d have no pinned supply; failing fetch "
|
|
"immediately",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
round_seq,
|
|
len(keys),
|
|
len(req.demanded),
|
|
)
|
|
self._finalize_outbound(kv_request_id, round_seq, success=False)
|
|
return
|
|
if result.local_idxs:
|
|
self._submit_transfer(kv_request_id, result, req, round_seq)
|
|
# Prefiller-first mode: finish_request may have run before
|
|
# fetch arrived. If so, finalize once we know what was
|
|
# demanded — fully satisfied → success, else early-fail.
|
|
if req.finishing and req.inflight == 0:
|
|
self._finalize_outbound(kv_request_id, round_seq)
|
|
|
|
def on_abort_fetch(self, kv_request_id: str, round_seq: int = 0) -> None:
|
|
"""Handle an AbortFetchMsg from the peer, cancelling one round."""
|
|
# Abort for an unknown id may be a benign race/duplicate or a
|
|
# real protocol violation; we don't track completed ids, so warn.
|
|
st = self._requests.get(kv_request_id)
|
|
if (st is None or not st.outbound) and not self._has_inflight_for(
|
|
kv_request_id
|
|
):
|
|
logger.warning(
|
|
"P2PSession %s: abort_fetch for unknown kv_request_id=%s "
|
|
"(no outbound or inflight state); benign race or stale",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
)
|
|
# Idempotent: receiving AbortFetchMsg again before we've sent the
|
|
# ack just triggers another drain attempt without resetting the
|
|
# deadline.
|
|
self._get_or_create_request(kv_request_id)
|
|
self._pending_aborts.setdefault((kv_request_id, round_seq), time.monotonic())
|
|
self._drain_abort(kv_request_id, round_seq)
|
|
|
|
def on_lookup(
|
|
self,
|
|
kv_request_id: str,
|
|
keys: Sequence[OffloadKey],
|
|
round_seq: int = 0,
|
|
) -> None:
|
|
"""Enqueue a LookupMsg from a symmetric-P2P consumer.
|
|
|
|
Dispatch runs during ``session.poll()`` where the
|
|
:class:`ParentManager` handle is not available, so this only
|
|
records the raw request. It is resolved — querying the tiering
|
|
manager and emitting the aggregated ``LookupRespMsg`` — by the
|
|
next :meth:`serve_external_requests`, the sole window in which
|
|
parent calls are valid.
|
|
"""
|
|
logger.debug(
|
|
"P2P LOOKUP server %s: RECV LookupMsg kv_request_id=%s round=%s keys=%d",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
round_seq,
|
|
len(keys),
|
|
)
|
|
self._get_or_create_request(kv_request_id).pending_lookups.append(
|
|
_PendingLookup(
|
|
keys=list(keys),
|
|
enqueued_at=time.monotonic(),
|
|
round_seq=round_seq,
|
|
)
|
|
)
|
|
self._serve_pending.add(kv_request_id)
|
|
|
|
def serve_external_requests(self, parent: ParentManager) -> None:
|
|
"""Resolve inbound peer lookups against the tiering manager.
|
|
|
|
Called once per scheduler step with a ``parent`` handle valid
|
|
only for this call. Drains newly-enqueued LookupMsgs, re-polls
|
|
any parked HIT_PENDING / RETRY keys, and releases the
|
|
bookkeeping for lookups closed since the last serve.
|
|
"""
|
|
for kv_request_id in list(self._serve_pending):
|
|
st = self._requests.get(kv_request_id)
|
|
if st is None:
|
|
self._serve_pending.discard(kv_request_id)
|
|
continue
|
|
if st.pending_lookups:
|
|
pending = st.pending_lookups
|
|
st.pending_lookups = []
|
|
for pl in pending:
|
|
self._process_inbound_lookup(
|
|
kv_request_id, pl.keys, pl.enqueued_at, pl.round_seq, parent
|
|
)
|
|
self._resolve_pending_lookups(kv_request_id, parent)
|
|
st = self._requests.get(kv_request_id)
|
|
if st is None or (not st.pending_lookups and not st.lookups):
|
|
self._serve_pending.discard(kv_request_id)
|
|
self._maybe_prune(kv_request_id)
|
|
|
|
if self._finished_lookup_ctxs:
|
|
for ctx in self._finished_lookup_ctxs:
|
|
parent.on_request_finished(ctx)
|
|
self._finished_lookup_ctxs = []
|
|
|
|
def _poll_lookup_keys(
|
|
self,
|
|
lookup: _ActiveLookup,
|
|
keys: Iterable[OffloadKey],
|
|
parent: ParentManager,
|
|
) -> list[OffloadKey]:
|
|
"""Poll ``keys`` against the tiering manager and pin any HITs.
|
|
|
|
For each key not already definitively resolved, query
|
|
``parent.lookup`` and record the outcome on ``lookup``: HIT / MISS
|
|
land in ``resolved`` and clear ``pending``; HIT_PENDING / RETRY park
|
|
in ``pending`` for a later serve. Newly-HIT keys are pinned in one
|
|
batch via :meth:`_pin_and_register_hits` and also returned (for
|
|
caller logging).
|
|
|
|
Shared by the first-sighting pass (:meth:`_process_inbound_lookup`)
|
|
and the re-poll pass (:meth:`_resolve_pending_lookups`); callers pass
|
|
a de-duplicated ``keys`` collection.
|
|
"""
|
|
new_hits: list[OffloadKey] = []
|
|
for h in keys:
|
|
if h in lookup.resolved:
|
|
continue
|
|
result = parent.lookup(h, lookup.ctx)
|
|
if result is LookupResult.HIT:
|
|
new_hits.append(h)
|
|
lookup.resolved[h] = True
|
|
lookup.pending.discard(h)
|
|
elif result is LookupResult.MISS:
|
|
lookup.resolved[h] = False
|
|
lookup.pending.discard(h)
|
|
else:
|
|
lookup.pending.add(h)
|
|
if new_hits:
|
|
self._pin_and_register_hits(lookup, new_hits, parent)
|
|
return new_hits
|
|
|
|
def _process_inbound_lookup(
|
|
self,
|
|
kv_request_id: str,
|
|
keys: list[OffloadKey],
|
|
enqueued_at: float,
|
|
round_seq: int,
|
|
parent: ParentManager,
|
|
) -> None:
|
|
"""Resolve one enqueued LookupMsg against ``parent``.
|
|
|
|
For each key, query the tiering manager via ``parent.lookup``
|
|
and pin any HITs immediately via ``parent.create_store_job``
|
|
(plumbed into the existing ``add_stored_blocks`` matching path so
|
|
the eventual FetchMsg finds them). HIT_PENDING / RETRY keys park
|
|
in :class:`_ActiveLookup` for re-polling by
|
|
:meth:`_resolve_pending_lookups`.
|
|
|
|
The outbound ``LookupRespMsg`` is deferred until every key has
|
|
settled to HIT or MISS (or the batch-level ``deadline`` fires,
|
|
forcing any stragglers to MISS). One LookupRespMsg goes out per
|
|
LookupMsg — carrying every key in wire order — after which
|
|
``parent.on_request_finished`` fires and the entry is dropped.
|
|
"""
|
|
self._lookup_id_counter += 1
|
|
lookup_id = self._lookup_id_counter
|
|
ctx = ReqContext(req_id=f"p2p:{self._peer_id}:{kv_request_id}:lu{lookup_id}")
|
|
lookup = _ActiveLookup(
|
|
lookup_id=lookup_id,
|
|
kv_request_id=kv_request_id,
|
|
ctx=ctx,
|
|
round_seq=round_seq,
|
|
keys=list(keys),
|
|
deadline=enqueued_at + _LOOKUP_PENDING_TIMEOUT_S,
|
|
)
|
|
|
|
# Open per-request bookkeeping for this synthetic ctx before the
|
|
# first lookup; released by ``on_request_finished`` once every
|
|
# key has settled.
|
|
parent.on_new_request(ctx)
|
|
|
|
# dict.fromkeys de-duplicates keys within the LookupMsg while
|
|
# preserving wire order — each unique key is polled once.
|
|
hit_keys = self._poll_lookup_keys(lookup, dict.fromkeys(lookup.keys), parent)
|
|
|
|
logger.debug(
|
|
"P2P LOOKUP server %s: RESOLVED kv_request_id=%s hits=%d misses=%d "
|
|
"pending=%d",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
len(hit_keys),
|
|
sum(1 for v in lookup.resolved.values() if not v),
|
|
len(lookup.pending),
|
|
)
|
|
|
|
if lookup.pending:
|
|
self._get_or_create_request(kv_request_id).lookups[lookup_id] = lookup
|
|
else:
|
|
# Every key resolved on first sight — emit the aggregated
|
|
# response now and close the synthetic request.
|
|
self._finalize_lookup(lookup, parent)
|
|
|
|
def _pin_and_register_hits(
|
|
self,
|
|
lookup: _ActiveLookup,
|
|
keys: list[OffloadKey],
|
|
parent: ParentManager,
|
|
) -> None:
|
|
"""Pin primary slots for HIT keys and park them as the lookup's
|
|
round supply via ``add_stored_blocks``.
|
|
|
|
Caller has already confirmed every key is HIT, so the JobMetadata
|
|
returned by ``parent.create_store_job`` carries parallel
|
|
``keys``/``block_ids`` of length ``len(keys)``. Nothing can evict
|
|
between that confirmation and the pin because the whole
|
|
``serve_external_requests`` call runs inside one hold of the tiering
|
|
manager's lock -- which is why that call must never release it
|
|
partway through.
|
|
"""
|
|
meta = parent.create_store_job(keys, lookup.ctx)
|
|
self.add_stored_blocks(
|
|
lookup.kv_request_id,
|
|
list(meta.keys),
|
|
meta.chunk_ids.tolist(),
|
|
meta.job_id,
|
|
round_seq=lookup.round_seq,
|
|
from_lookup=True,
|
|
)
|
|
|
|
def _resolve_pending_lookups(
|
|
self, kv_request_id: str, parent: ParentManager
|
|
) -> None:
|
|
"""Re-poll a request's deferred LookupMsg keys; finalize when ready.
|
|
|
|
Walks every parked :class:`_ActiveLookup` for ``kv_request_id`` and
|
|
re-calls ``parent.lookup`` per still-pending key, moving HIT/MISS
|
|
results into ``resolved``. If ``deadline`` has passed, remaining
|
|
``pending`` keys are force-resolved to MISS so the consumer
|
|
can fall back instead of waiting on a stuck producer. Newly-HIT
|
|
keys are pinned via ``parent.create_store_job`` in one call per
|
|
affected lookup. Lookups whose ``pending`` empties out get their
|
|
aggregated LookupRespMsg sent by :meth:`_finalize_lookup`.
|
|
"""
|
|
st = self._requests.get(kv_request_id)
|
|
if st is None or not st.lookups:
|
|
return
|
|
now = time.monotonic()
|
|
finished_lookups: list[int] = []
|
|
for lookup_id, lookup in st.lookups.items():
|
|
self._poll_lookup_keys(lookup, list(lookup.pending), parent)
|
|
|
|
if lookup.pending and now >= lookup.deadline:
|
|
for h in lookup.pending:
|
|
lookup.resolved[h] = False
|
|
lookup.pending.clear()
|
|
|
|
if not lookup.pending:
|
|
finished_lookups.append(lookup_id)
|
|
|
|
for lookup_id in finished_lookups:
|
|
lookup = st.lookups.pop(lookup_id)
|
|
self._finalize_lookup(lookup, parent)
|
|
|
|
def _finalize_lookup(self, lookup: _ActiveLookup, parent: ParentManager) -> None:
|
|
"""Emit the aggregated LookupRespMsg and close the synthetic request.
|
|
|
|
Called once per lookup when ``pending`` is empty — either every
|
|
key resolved to HIT or MISS, or the deadline forced remaining
|
|
stragglers to MISS. Preserves the wire order of the inbound
|
|
LookupMsg so the client can zip keys and hits positionally.
|
|
"""
|
|
if lookup.keys:
|
|
hits = [lookup.resolved[h] for h in lookup.keys]
|
|
n_hit = sum(1 for v in hits if v)
|
|
logger.debug(
|
|
"P2P LOOKUP server %s: SEND LookupRespMsg kv_request_id=%s "
|
|
"keys=%d hits=%d misses=%d",
|
|
self._peer_id,
|
|
lookup.kv_request_id,
|
|
len(lookup.keys),
|
|
n_hit,
|
|
len(hits) - n_hit,
|
|
)
|
|
self._send(
|
|
{
|
|
TYPE_KEY: LookupRespMsg.TYPE,
|
|
LookupRespMsg.KV_REQUEST_ID: lookup.kv_request_id,
|
|
LookupRespMsg.KEYS: list(lookup.keys),
|
|
LookupRespMsg.HITS: hits,
|
|
}
|
|
)
|
|
parent.on_request_finished(lookup.ctx)
|
|
|
|
def _finish_inbound_lookups(self, kv_request_id: str) -> None:
|
|
"""Close the server-side lookup phase for ``kv_request_id``.
|
|
|
|
Pops every parked ``_ActiveLookup`` for this id (so
|
|
``_resolve_pending_lookups`` cannot promote a HIT_PENDING /
|
|
RETRY key into a fresh ``parent.create_store_job`` after this
|
|
point) and queues each ``lookup.ctx`` for
|
|
``parent.on_request_finished`` (fired by the next
|
|
``serve_external_requests``, since no parent handle is available
|
|
during dispatch) so the TieringManager can release per-lookup
|
|
bookkeeping. Any still-unprocessed raw LookupMsg for this id is
|
|
dropped — it never got ``on_new_request``, so nothing is owed.
|
|
The aggregated LookupRespMsg is skipped — the client already
|
|
knows the request is over (it just sent a terminal FetchMsg, or
|
|
is finishing locally).
|
|
|
|
Called on the two events that mean "no more lookup traffic for
|
|
``kv_request_id`` is expected on this session": the terminal
|
|
empty FetchMsg from the peer and a local ``finish``. Whichever
|
|
fires second is a no-op.
|
|
"""
|
|
st = self._requests.get(kv_request_id)
|
|
if st is None:
|
|
return
|
|
st.pending_lookups.clear()
|
|
for lookup in st.lookups.values():
|
|
self._finished_lookup_ctxs.append(lookup.ctx)
|
|
st.lookups.clear()
|
|
self._serve_pending.discard(kv_request_id)
|
|
self._maybe_prune(kv_request_id)
|
|
|
|
def finish(self, kv_request_id: str) -> None:
|
|
"""Mark an outbound request finishing.
|
|
|
|
No more submit_store calls will arrive for this id. Any blocks
|
|
the peer demanded but we never stored will never come; tell the
|
|
peer to stop waiting (TransferDoneMsg success=False) instead of
|
|
letting it hit _LOAD_TIMEOUT_S.
|
|
|
|
Also drops any in-flight lookups for this kv_request_id and
|
|
queues their ctxs for ``parent.on_request_finished`` so the
|
|
TieringManager can release per-request bookkeeping. (For
|
|
symmetric P2P this path is rarely hit since the producer has no
|
|
local request lifecycle for the consumer's id; this cleanup is
|
|
mostly active on the PD side.)
|
|
|
|
If the decoder hasn't sent fetch yet (no demand received),
|
|
defer — on_fetch will finalize once demand arrives.
|
|
|
|
If inflight transfers exist for this id, defer — the last
|
|
completing transfer in collect_results will fire the message.
|
|
"""
|
|
self._finish_inbound_lookups(kv_request_id)
|
|
|
|
st = self._requests.get(kv_request_id)
|
|
if st is None:
|
|
return
|
|
for key, req in list(st.outbound.items()):
|
|
req.finishing = True
|
|
if not req.demand_received or req.inflight:
|
|
# No demand yet (prefiller-first): on_fetch finalizes via
|
|
# `finishing`. Inflight: the last completion finalizes.
|
|
continue
|
|
self._finalize_outbound(kv_request_id, key)
|
|
|
|
def collect_results(self) -> list[StoreResult]:
|
|
"""Drain timeouts, deferred results, and transport completions.
|
|
|
|
Inbound LookupMsg resolution (including re-polling HIT_PENDING /
|
|
RETRY keys) is NOT done here — it runs in
|
|
``serve_external_requests`` where the ParentManager is available.
|
|
"""
|
|
results: list[StoreResult] = self._timeout_pending_store_jobs()
|
|
|
|
# Scope the poll to this peer: the transport is shared across all peer
|
|
# sessions of the engine, and poll() drains completed handles. An
|
|
# unscoped poll here would consume sibling sessions' completions and
|
|
# report them as "unknown transfer_id", starving those sessions.
|
|
poll_result = self._transport.poll(self._peer_id)
|
|
|
|
for tid in poll_result.done:
|
|
xfer = self._inflight_pop(tid)
|
|
if xfer is None:
|
|
# Bug signal: transport reported a transfer we have no
|
|
# bookkeeping for. Likely a double-completion in the
|
|
# transport or a stale removal in the session. The
|
|
# attached job(s) still live in _store_jobs and will be
|
|
# surfaced as failures by _timeout_pending_store_jobs
|
|
# after _STORE_TIMEOUT_S, but log loudly so the
|
|
# underlying bug is findable.
|
|
logger.error(
|
|
"P2PSession %s: transport reported done for unknown "
|
|
"transfer_id=%d; attached job(s) will fail via "
|
|
"store-timeout instead of completing now",
|
|
self._peer_id,
|
|
tid,
|
|
)
|
|
continue
|
|
results.extend(
|
|
self._settle_xfer_jobs(xfer, success=not xfer.round.cancelled)
|
|
)
|
|
rnd = xfer.round
|
|
st = self._requests.get(xfer.kv_request_id)
|
|
if st is not None and st.outbound.get(xfer.round_key) is rnd:
|
|
rnd.remaining -= xfer.block_count
|
|
assert rnd.remaining >= 0, (
|
|
f"remaining went negative for kv_request_id={xfer.kv_request_id}"
|
|
)
|
|
if rnd.cancelled:
|
|
if rnd.inflight == 0:
|
|
self._finalize_cancelled(xfer.kv_request_id, xfer.round_key)
|
|
elif rnd.remaining == 0:
|
|
self._finalize_outbound(
|
|
xfer.kv_request_id, xfer.round_key, success=True
|
|
)
|
|
elif rnd.finishing and rnd.inflight == 0:
|
|
self._finalize_outbound(
|
|
xfer.kv_request_id, xfer.round_key, success=False
|
|
)
|
|
self._maybe_prune(xfer.kv_request_id)
|
|
|
|
for tid in poll_result.failed:
|
|
xfer = self._inflight_pop(tid)
|
|
if xfer is None:
|
|
# See the matching error log in the done branch above.
|
|
logger.error(
|
|
"P2PSession %s: transport reported failed for unknown "
|
|
"transfer_id=%d; attached job(s) will fail via "
|
|
"store-timeout instead of completing now",
|
|
self._peer_id,
|
|
tid,
|
|
)
|
|
continue
|
|
rnd = xfer.round
|
|
st = self._requests.get(xfer.kv_request_id)
|
|
in_round = st is not None and st.outbound.get(xfer.round_key) is rnd
|
|
if in_round and not rnd.cancelled:
|
|
# One failed transfer fails the round, but its other
|
|
# transfers may still be writing. Cancel them like a
|
|
# timed-out round: their pins and the TransferDoneMsg wait
|
|
# until they are terminal.
|
|
rnd.cancelled = True
|
|
results.extend(self._fail_unwritten_round_jobs(rnd))
|
|
results.extend(self._settle_xfer_jobs(xfer, success=False))
|
|
if in_round and rnd.inflight == 0:
|
|
self._finalize_cancelled(xfer.kv_request_id, xfer.round_key)
|
|
self._maybe_prune(xfer.kv_request_id)
|
|
|
|
# Finalizing transfers can queue results for supplied but unfetched blocks.
|
|
if self._pending_store_results:
|
|
results.extend(self._pending_store_results)
|
|
self._pending_store_results.clear()
|
|
|
|
return results
|
|
|
|
def collect_idle_timeouts(self) -> list[StoreResult]:
|
|
"""Run only the store-job timeout sweep.
|
|
|
|
Used by the coordinator's no-conn poll path: a pending session
|
|
cannot have inflight transfers (no peer registered yet), so we
|
|
skip the transport poll and the deferred-result drain.
|
|
"""
|
|
return self._timeout_pending_store_jobs()
|
|
|
|
def drain_pending_aborts(self) -> None:
|
|
"""Re-attempt every parked abort once per poll tick."""
|
|
for kv_request_id, round_seq in list(self._pending_aborts):
|
|
self._drain_abort(kv_request_id, round_seq)
|
|
|
|
def close(self) -> tuple[list[int], list[ReqContext]]:
|
|
"""Tear down. Cancels inflight.
|
|
|
|
Returns ``(failed_store_job_ids, failed_serves)`` where
|
|
``failed_serves`` are synthetic lookup ctxs still owing a
|
|
``parent.on_request_finished``. The session is going away with no
|
|
parent handle in hand, so the manager flushes these in its next
|
|
``serve_external_requests``.
|
|
"""
|
|
failed_stores = list(self._store_jobs.keys())
|
|
self._store_jobs.clear()
|
|
if self._inflight:
|
|
self._transport.cancel(list(self._inflight.keys()))
|
|
self._inflight.clear()
|
|
self._pending_store_results.clear()
|
|
# Surface every synthetic ctx still owing on_request_finished so
|
|
# the manager can release the TieringManager's per-request
|
|
# bookkeeping: parked lookups plus any already queued from a
|
|
# FetchMsg / finish that closed them before this teardown.
|
|
failed_serves = [
|
|
lu.ctx for st in self._requests.values() for lu in st.lookups.values()
|
|
]
|
|
failed_serves.extend(self._finished_lookup_ctxs)
|
|
self._requests.clear()
|
|
self._serve_pending.clear()
|
|
self._pending_aborts.clear()
|
|
self._slow_aborts.clear()
|
|
self._finished_lookup_ctxs.clear()
|
|
return failed_stores, failed_serves
|
|
|
|
@property
|
|
def has_inflight_transfers(self) -> bool:
|
|
"""True if any outbound store transfer is still in flight."""
|
|
return bool(self._inflight)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Internal — inflight bookkeeping
|
|
# ------------------------------------------------------------------
|
|
|
|
def _has_inflight_for(self, kv_request_id: str) -> bool:
|
|
st = self._requests.get(kv_request_id)
|
|
return st is not None and bool(st.inflight_tids)
|
|
|
|
def _inflight_add(self, tid: int, xfer: _InflightXfer) -> None:
|
|
"""Insert an inflight transfer and record it on the request."""
|
|
self._inflight[tid] = xfer
|
|
self._get_or_create_request(xfer.kv_request_id).inflight_tids.add(tid)
|
|
|
|
def _inflight_pop(self, tid: int) -> _InflightXfer | None:
|
|
"""Pop an inflight transfer and drop it from the request's set.
|
|
|
|
Callers are responsible for the ``_maybe_prune`` that may follow once
|
|
the request's other state has also cleared.
|
|
"""
|
|
xfer = self._inflight.pop(tid, None)
|
|
if xfer is None:
|
|
return None
|
|
xfer.round.inflight -= 1
|
|
assert xfer.round.inflight >= 0
|
|
st = self._requests.get(xfer.kv_request_id)
|
|
if st is not None:
|
|
st.inflight_tids.discard(tid)
|
|
return xfer
|
|
|
|
def _settle_xfer_jobs(
|
|
self, xfer: _InflightXfer, success: bool
|
|
) -> list[StoreResult]:
|
|
"""Emit StoreResults for a completed transfer's store jobs.
|
|
|
|
Pops each attached job from ``_store_jobs`` and clears it from
|
|
its round's pending set. A job already popped (via timeout,
|
|
cancel, etc.) is skipped so we never double-emit a contradictory
|
|
result.
|
|
"""
|
|
results: list[StoreResult] = []
|
|
still_pinned = self._read_job_ids() if xfer.round.cancelled else set()
|
|
for job_id in xfer.job_ids:
|
|
if job_id in still_pinned:
|
|
continue
|
|
if self._store_jobs.pop(job_id, None) is None:
|
|
continue
|
|
results.append(StoreResult(job_id=job_id, success=success))
|
|
xfer.round.pending_job_ids.discard(job_id)
|
|
return results
|
|
|
|
# ------------------------------------------------------------------
|
|
# Internal — finalize / abort drain
|
|
# ------------------------------------------------------------------
|
|
|
|
def _fail_round_jobs(self, rnd: _OutboundRequestState) -> list[StoreResult]:
|
|
"""Fail a terminated round's still-pending store jobs (idempotent)."""
|
|
results: list[StoreResult] = []
|
|
for job_id in rnd.pending_job_ids:
|
|
if self._store_jobs.pop(job_id, None) is None:
|
|
continue
|
|
results.append(StoreResult(job_id=job_id, success=False))
|
|
rnd.pending_job_ids.clear()
|
|
return results
|
|
|
|
def _finalize_cancelled(self, kv_request_id: str, round_key: int) -> None:
|
|
"""Finalize a cancelled round once its last transfer is terminal.
|
|
|
|
An aborted round gets no TransferDoneMsg: its parked abort answers
|
|
the peer with AbortAckMsg on the next drain.
|
|
"""
|
|
self._finalize_outbound(
|
|
kv_request_id,
|
|
round_key,
|
|
success=False,
|
|
send_done=(kv_request_id, round_key) not in self._pending_aborts,
|
|
)
|
|
|
|
def _read_job_ids(self, rnd: _OutboundRequestState | None = None) -> set[int]:
|
|
"""Jobs whose blocks an inflight transfer (of ``rnd``, if given)
|
|
still reads; their pins must be held."""
|
|
return {
|
|
jid
|
|
for xfer in self._inflight.values()
|
|
if rnd is None or xfer.round is rnd
|
|
for jid in xfer.job_ids
|
|
}
|
|
|
|
def _fail_unwritten_round_jobs(
|
|
self, rnd: _OutboundRequestState
|
|
) -> list[StoreResult]:
|
|
"""Fail a cancelled round's jobs that no inflight transfer carries.
|
|
|
|
Jobs that an inflight transfer carries keep their pins; they settle
|
|
when that transfer turns terminal (``_settle_xfer_jobs``).
|
|
"""
|
|
written = self._read_job_ids(rnd)
|
|
results: list[StoreResult] = []
|
|
for job_id in list(rnd.pending_job_ids):
|
|
if job_id in written:
|
|
continue
|
|
rnd.pending_job_ids.discard(job_id)
|
|
if self._store_jobs.pop(job_id, None) is None:
|
|
continue
|
|
results.append(StoreResult(job_id=job_id, success=False))
|
|
return results
|
|
|
|
def _finalize_outbound(
|
|
self,
|
|
kv_request_id: str,
|
|
round_key: int,
|
|
success: bool | None = None,
|
|
send_done: bool = True,
|
|
) -> None:
|
|
"""Pop one round and emit its terminal results.
|
|
|
|
If ``success`` is None, derive it from ``req.remaining == 0``.
|
|
``send_done=False`` skips the TransferDoneMsg (terminal empty
|
|
fetch). Other rounds of the id are untouched.
|
|
"""
|
|
st = self._requests[kv_request_id]
|
|
req = st.outbound[round_key]
|
|
if req.cancelled and req.inflight:
|
|
return
|
|
del st.outbound[round_key]
|
|
if req.cancelled:
|
|
success = False
|
|
if success is None:
|
|
success = req.demand_received and req.remaining == 0
|
|
settled = self._fail_round_jobs(req) if not success else None
|
|
if settled is not None:
|
|
self._pending_store_results.extend(settled)
|
|
else:
|
|
for job_id in req.pending_job_ids:
|
|
if self._store_jobs.pop(job_id, None) is None:
|
|
continue
|
|
self._pending_store_results.append(
|
|
StoreResult(job_id=job_id, success=True)
|
|
)
|
|
req.pending_job_ids.clear()
|
|
logger.debug(
|
|
"P2PSession %s: finalize kv_request_id=%s round=%s success=%s "
|
|
"remaining=%d leftover_available=%d send_done=%s",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
round_key,
|
|
success,
|
|
req.remaining,
|
|
len(req.available),
|
|
send_done,
|
|
)
|
|
if send_done and req.demand_received:
|
|
self._send(
|
|
{
|
|
TYPE_KEY: TransferDoneMsg.TYPE,
|
|
TransferDoneMsg.KV_REQUEST_ID: kv_request_id,
|
|
TransferDoneMsg.SUCCESS: success,
|
|
TransferDoneMsg.ROUND_SEQ: round_key,
|
|
}
|
|
)
|
|
self._maybe_prune(kv_request_id)
|
|
|
|
def _drain_abort(self, kv_request_id: str, round_seq: int) -> None:
|
|
"""One drain attempt for a pending abort.
|
|
|
|
Cancels the aborted round, then asks the transport to cancel its
|
|
inflight transfers in ``mode="wait"``. Sends ``AbortAckMsg`` only
|
|
once none of them can still write to the peer: the ack lets the
|
|
peer reuse the destination blocks. A transfer that NIXL does not
|
|
report terminal is never force-released, however long the drain
|
|
takes: neither releasing its handle nor removing the remote agent
|
|
stops a WRITE that is already in flight. Past
|
|
``_CANCEL_DRAIN_TIMEOUT_S`` the drain only logs a warning.
|
|
"""
|
|
st = self._requests[kv_request_id]
|
|
rnd = st.outbound.get(round_seq)
|
|
if rnd is not None and not rnd.cancelled:
|
|
# Fail the round's jobs now instead of leaking them to the
|
|
# store timeout, but keep the pins of those still being written.
|
|
rnd.cancelled = True
|
|
self._pending_store_results.extend(self._fail_unwritten_round_jobs(rnd))
|
|
ids = [
|
|
tid
|
|
for tid, x in self._inflight.items()
|
|
if x.kv_request_id == kv_request_id and x.round_key == round_seq
|
|
]
|
|
if not ids:
|
|
if rnd is not None and st.outbound.get(round_seq) is rnd:
|
|
self._finalize_outbound(
|
|
kv_request_id, round_seq, success=False, send_done=False
|
|
)
|
|
self._finalize_abort(kv_request_id, round_seq)
|
|
return
|
|
|
|
now = time.monotonic()
|
|
key = (kv_request_id, round_seq)
|
|
if (
|
|
now - self._pending_aborts[key] >= _CANCEL_DRAIN_TIMEOUT_S
|
|
and key not in self._slow_aborts
|
|
):
|
|
self._slow_aborts.add(key)
|
|
logger.warning(
|
|
"P2PSession %s: abort of kv_request_id=%s round=%s still "
|
|
"waiting after %.0fs for %d transfers to stop; holding their "
|
|
"pins and the AbortAck until NIXL reports them terminal",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
round_seq,
|
|
_CANCEL_DRAIN_TIMEOUT_S,
|
|
len(ids),
|
|
)
|
|
|
|
# All of the aborted round's transfers share one round state.
|
|
if not self._cancel_due(self._inflight[ids[0]].round, now):
|
|
return
|
|
still = self._transport.cancel(ids, mode="wait")
|
|
# Tids the transport successfully released are gone from its
|
|
# _inflight; mirror that in session bookkeeping so they don't
|
|
# block the drain forever waiting for a poll() event that will
|
|
# never come.
|
|
still_set = set(still)
|
|
for tid in ids:
|
|
if tid not in still_set:
|
|
self._inflight_pop(tid)
|
|
if not still:
|
|
if rnd is not None and st.outbound.get(round_seq) is rnd:
|
|
self._finalize_outbound(
|
|
kv_request_id, round_seq, success=False, send_done=False
|
|
)
|
|
self._finalize_abort(kv_request_id, round_seq)
|
|
|
|
@staticmethod
|
|
def _cancel_due(rnd: _OutboundRequestState, now: float) -> bool:
|
|
"""Rate-limit wait-mode cancels of ``rnd``'s transfers.
|
|
|
|
Shared by the store-timeout sweep and the abort drain, so a round
|
|
gets at most one attempt per ``_CANCEL_RETRY_INTERVAL_S``.
|
|
"""
|
|
if (
|
|
rnd.last_cancel_at is not None
|
|
and now - rnd.last_cancel_at < _CANCEL_RETRY_INTERVAL_S
|
|
):
|
|
return False
|
|
rnd.last_cancel_at = now
|
|
return True
|
|
|
|
def _finalize_abort(self, kv_request_id: str, round_seq: int) -> None:
|
|
self._pending_aborts.pop((kv_request_id, round_seq), None)
|
|
self._slow_aborts.discard((kv_request_id, round_seq))
|
|
self._send(
|
|
{
|
|
TYPE_KEY: AbortAckMsg.TYPE,
|
|
AbortAckMsg.KV_REQUEST_ID: kv_request_id,
|
|
AbortAckMsg.ROUND_SEQ: round_seq,
|
|
}
|
|
)
|
|
self._maybe_prune(kv_request_id)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Internal — transfers and store-job timeouts
|
|
# ------------------------------------------------------------------
|
|
|
|
def _submit_transfer(
|
|
self,
|
|
kv_request_id: str,
|
|
result: _MatchResult,
|
|
rnd: _OutboundRequestState,
|
|
round_key: int,
|
|
) -> None:
|
|
logger.debug(
|
|
"P2PSession %s: NIXL write_blocks CALL kv_request_id=%s "
|
|
"local_idxs=%d remote_idxs=%d",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
len(result.local_idxs),
|
|
len(result.remote_idxs),
|
|
)
|
|
transfer_id = self._transport.write_blocks(
|
|
self._peer_id, result.local_idxs, result.remote_idxs
|
|
)
|
|
if transfer_id is not None:
|
|
logger.debug(
|
|
"P2PSession %s: NIXL write_blocks SUBMITTED kv_request_id=%s "
|
|
"transfer_id=%d blocks=%d",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
transfer_id,
|
|
len(result.local_idxs),
|
|
)
|
|
rnd.inflight += 1
|
|
self._inflight_add(
|
|
transfer_id,
|
|
_InflightXfer(
|
|
kv_request_id=kv_request_id,
|
|
block_count=len(result.local_idxs),
|
|
job_ids=result.job_ids,
|
|
round=rnd,
|
|
round_key=round_key,
|
|
),
|
|
)
|
|
else:
|
|
logger.warning(
|
|
"P2PSession %s: write_blocks failed for %s (%d blocks)",
|
|
self._peer_id,
|
|
kv_request_id,
|
|
len(result.local_idxs),
|
|
)
|
|
# The matched blocks were popped from rnd.demanded /
|
|
# rnd.available, but no inflight will satisfy them, so
|
|
# remaining will never reach 0 on its own. Mark the round
|
|
# as finishing so the existing terminal paths clean up: if
|
|
# other transfers of this round are in flight, the last one
|
|
# to drain will fire _finalize_outbound(success=False) via
|
|
# the elif branch in collect_results. If nothing else is in
|
|
# flight, finalize now so the peer and the local store jobs
|
|
# don't wait for finish_request or for _STORE_TIMEOUT_S /
|
|
# _LOAD_TIMEOUT_S.
|
|
rnd.finishing = True
|
|
st = self._requests.get(kv_request_id)
|
|
if (
|
|
st is not None
|
|
and st.outbound.get(round_key) is rnd
|
|
and rnd.inflight == 0
|
|
):
|
|
self._finalize_outbound(kv_request_id, round_key, success=False)
|
|
|
|
def _timeout_pending_store_jobs(self) -> list[StoreResult]:
|
|
if not self._store_jobs:
|
|
return []
|
|
now = time.monotonic()
|
|
deadline = now - _STORE_TIMEOUT_S
|
|
timed_out: list[int] | None = None
|
|
for jid, submitted_at in self._store_jobs.items():
|
|
if submitted_at <= deadline:
|
|
if timed_out is None:
|
|
timed_out = []
|
|
timed_out.append(jid)
|
|
if timed_out is None:
|
|
if not any(xfer.round.cancelled for xfer in self._inflight.values()):
|
|
return []
|
|
timed_out = []
|
|
results: list[StoreResult] = []
|
|
expired = set(timed_out)
|
|
rounds = {
|
|
(xfer.kv_request_id, xfer.round_key): xfer.round
|
|
for xfer in self._inflight.values()
|
|
if xfer.round.cancelled or expired.intersection(xfer.job_ids)
|
|
}
|
|
for (kv_request_id, round_key), rnd in rounds.items():
|
|
rnd.cancelled = True
|
|
if not self._cancel_due(rnd, now):
|
|
continue
|
|
tids = [tid for tid, xfer in self._inflight.items() if xfer.round is rnd]
|
|
still = set(self._transport.cancel(tids, mode="wait"))
|
|
for tid in tids:
|
|
if tid in still:
|
|
continue
|
|
xfer = self._inflight_pop(tid)
|
|
assert xfer is not None
|
|
results.extend(self._settle_xfer_jobs(xfer, success=False))
|
|
st = self._requests.get(kv_request_id)
|
|
if (
|
|
rnd.inflight == 0
|
|
and st is not None
|
|
and st.outbound.get(round_key) is rnd
|
|
):
|
|
self._finalize_cancelled(kv_request_id, round_key)
|
|
|
|
# Cancellation may still be draining. Keep every affected pin until
|
|
# the transport confirms that no transfer reads its primary slot.
|
|
inflight_job_ids = self._read_job_ids()
|
|
for jid in timed_out:
|
|
if jid in inflight_job_ids:
|
|
continue
|
|
if self._store_jobs.pop(jid, None) is None:
|
|
continue
|
|
results.append(StoreResult(job_id=jid, success=False))
|
|
logger.warning("P2PSession %s: store job %d timed out", self._peer_id, jid)
|
|
# A failed store job's primary slots are unpinned by the engine, so its
|
|
# blocks must no longer be offered to a fetch. A symmetric round then
|
|
# fails fast in on_fetch instead of reading a slot that may be reused.
|
|
expired = set(timed_out)
|
|
for st in self._requests.values():
|
|
for rnd in st.outbound.values():
|
|
if rnd.pending_job_ids & expired:
|
|
rnd.pending_job_ids -= expired
|
|
rnd.available = {
|
|
key: entry
|
|
for key, entry in rnd.available.items()
|
|
if entry[0] not in expired
|
|
}
|
|
return results
|