[Bugfix][KV Offload] Bound unanswered P2P lookup waits (#56836)
Signed-off-by: jacklin78911-collab <jacklin78911@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Codex
Robert Shaw
Itay Etelis
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Or Ozeri
Kevin H. Luu
parent
81e1c3f183
commit
3f1bcfd82b
@@ -32,6 +32,7 @@ logger = init_logger(__name__)
|
||||
|
||||
_LOAD_TIMEOUT_S = 30.0
|
||||
_ABORT_ACK_TIMEOUT_S = 10.0
|
||||
_LOOKUP_TIMEOUT_S = 30.0
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -59,10 +60,8 @@ class _ClientRequestState:
|
||||
|
||||
# -- Lookup phase (symmetric P2P only; untouched for PD) --
|
||||
# Probe outcome per OffloadKey: None while in-flight (registered/sent
|
||||
# but unresolved), True/False once a LookupRespMsg lands. There is no
|
||||
# timeout — finish (via finish_request) is guaranteed after
|
||||
# the request's lookup() calls and clears every probe, so an
|
||||
# unanswered probe simply stays None until then.
|
||||
# but unresolved), True/False once a LookupRespMsg lands. Unanswered
|
||||
# probes expire the session so the manager can fall back to local compute.
|
||||
probes: dict[OffloadKey, bool | None] = field(default_factory=dict)
|
||||
# OffloadKeys registered but not yet flushed onto the wire. Drained and
|
||||
# cleared by the next flush_pending_lookups.
|
||||
@@ -136,6 +135,9 @@ class ClientRole:
|
||||
# Kept in exact sync with ``st.loads``.
|
||||
self._active_loads: set[str] = set()
|
||||
self._completed_loads: list[LoadResult] = []
|
||||
# Per-request progress deadlines. Only newly resolved probes extend
|
||||
# them; additional keys and repeated scheduler polls do not.
|
||||
self._lookup_deadlines: dict[str, float] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# State helpers
|
||||
@@ -172,6 +174,14 @@ class ClientRole:
|
||||
"""True if any kv_request_id has a fetch in flight."""
|
||||
return bool(self._active_loads)
|
||||
|
||||
def get_expired_lookup(self) -> tuple[str, float] | None:
|
||||
"""Return an expired request and seconds without progress, if any."""
|
||||
now = time.monotonic()
|
||||
for req_id, deadline in self._lookup_deadlines.items():
|
||||
if now >= deadline:
|
||||
return req_id, now - deadline + _LOOKUP_TIMEOUT_S
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
@@ -228,6 +238,7 @@ class ClientRole:
|
||||
)
|
||||
assert all(st.probes.get(key) is True for key in keys)
|
||||
st.probes.clear()
|
||||
self._lookup_deadlines.pop(kv_request_id, None)
|
||||
|
||||
def finish(self, kv_request_id: str) -> None:
|
||||
"""Finish a request: abort in-flight loads and release lookup state.
|
||||
@@ -272,6 +283,7 @@ class ClientRole:
|
||||
)
|
||||
st.probes.clear()
|
||||
st.unsent.clear()
|
||||
self._lookup_deadlines.pop(kv_request_id, None)
|
||||
self._flush_pending.discard(kv_request_id)
|
||||
self._maybe_prune(kv_request_id)
|
||||
|
||||
@@ -377,7 +389,7 @@ class ClientRole:
|
||||
)
|
||||
return None
|
||||
|
||||
def flush_pending_lookups(self) -> None:
|
||||
def flush_pending_lookups(self, send_ready: bool) -> None:
|
||||
"""Send a LookupMsg for each kv_request_id with unsent entries.
|
||||
|
||||
Called once per scheduler step from the manager's
|
||||
@@ -387,13 +399,14 @@ class ClientRole:
|
||||
keys, all tagged with the current round. register_lookup()
|
||||
de-dups in-flight and already-resolved (req_id, key) pairs, so
|
||||
each LookupMsg carries only the keys first probed in that step.
|
||||
Send-gating is handled by the injected ``_send`` callback
|
||||
(queues until ConnectAckMsg if needed).
|
||||
Wait for ConnectAckMsg before sending or starting the response deadline.
|
||||
|
||||
Only requests that registered new keys since the last flush are
|
||||
visited — the ``_flush_pending`` work-list avoids scanning every
|
||||
live request each scheduler step.
|
||||
"""
|
||||
if not send_ready:
|
||||
return
|
||||
for req_id in self._flush_pending:
|
||||
st = self._requests.get(req_id)
|
||||
if st is None or not st.unsent:
|
||||
@@ -401,6 +414,9 @@ class ClientRole:
|
||||
# The peer now holds lookup state for this id; finish owes a
|
||||
# terminal empty FetchMsg until a fetch closes it.
|
||||
st.peer_lookup_open = True
|
||||
self._lookup_deadlines.setdefault(
|
||||
req_id, time.monotonic() + _LOOKUP_TIMEOUT_S
|
||||
)
|
||||
logger.debug(
|
||||
"P2P LOOKUP client %s: SEND LookupMsg kv_request_id=%s keys=%d",
|
||||
self._peer_id,
|
||||
@@ -443,10 +459,16 @@ class ClientRole:
|
||||
st = self._requests.get(kv_request_id)
|
||||
if st is None:
|
||||
return
|
||||
progressed = False
|
||||
for h, hit in zip(keys, hits):
|
||||
key = OffloadKey(h)
|
||||
if key in st.probes:
|
||||
progressed |= st.probes[key] is None
|
||||
st.probes[key] = hit
|
||||
if all(hit is not None for hit in st.probes.values()):
|
||||
self._lookup_deadlines.pop(kv_request_id, None)
|
||||
elif progressed:
|
||||
self._lookup_deadlines[kv_request_id] = time.monotonic() + _LOOKUP_TIMEOUT_S
|
||||
|
||||
def collect_results(self) -> list[LoadResult]:
|
||||
"""Walk load timeouts and drain completed loads.
|
||||
@@ -454,9 +476,6 @@ class ClientRole:
|
||||
Loads past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg sent and
|
||||
enter the aborting phase. Aborting loads past
|
||||
``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed.
|
||||
|
||||
Lookups have no timeout: an unanswered probe stays None (RETRY)
|
||||
until finish_request clears it — see ``_ClientRequestState.probes``.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
to_remove: list[tuple[str, int]] = []
|
||||
@@ -525,4 +544,5 @@ class ClientRole:
|
||||
self._flush_pending.clear()
|
||||
self._active_loads.clear()
|
||||
self._completed_loads.clear()
|
||||
self._lookup_deadlines.clear()
|
||||
return ClientCloseResult(failed_jobs=failed_jobs, failed_req_ids=failed_req_ids)
|
||||
|
||||
@@ -755,10 +755,6 @@ class ServerRole:
|
||||
"""
|
||||
results: list[StoreResult] = self._timeout_pending_store_jobs()
|
||||
|
||||
if self._pending_store_results:
|
||||
results.extend(self._pending_store_results)
|
||||
self._pending_store_results.clear()
|
||||
|
||||
# Scope the poll to this peer: the transport is shared across all peer
|
||||
# sessions of the engine, and poll() drains completed handles. An
|
||||
# unscoped poll here would consume sibling sessions' completions and
|
||||
@@ -833,6 +829,11 @@ class ServerRole:
|
||||
self._finalize_cancelled(xfer.kv_request_id, xfer.round_key)
|
||||
self._maybe_prune(xfer.kv_request_id)
|
||||
|
||||
# Finalizing transfers can queue results for supplied but unfetched blocks.
|
||||
if self._pending_store_results:
|
||||
results.extend(self._pending_store_results)
|
||||
self._pending_store_results.clear()
|
||||
|
||||
return results
|
||||
|
||||
def collect_idle_timeouts(self) -> list[StoreResult]:
|
||||
|
||||
@@ -264,10 +264,9 @@ class P2PSession:
|
||||
"""Flush any aggregated symmetric-P2P lookups for this peer.
|
||||
|
||||
Called once per scheduler step from the manager's
|
||||
``on_schedule_end()``. Send-gating is handled inside the
|
||||
client's ``_send`` callback (queues until ConnectAckMsg).
|
||||
``on_schedule_end()``. Probes wait for ConnectAckMsg before sending.
|
||||
"""
|
||||
self._client.flush_pending_lookups()
|
||||
self._client.flush_pending_lookups(send_ready=self._send_ready)
|
||||
|
||||
def serve_external_requests(self, parent: ParentManager) -> None:
|
||||
"""Resolve inbound peer lookups against the tiering manager.
|
||||
@@ -329,6 +328,21 @@ class P2PSession:
|
||||
stores = self._server.collect_results()
|
||||
self._server.drain_pending_aborts()
|
||||
|
||||
# Drain buffered replies and completed transfers before expiring a peer.
|
||||
expired_lookup = self._client.get_expired_lookup()
|
||||
if expired_lookup is not None:
|
||||
req_id, elapsed = expired_lookup
|
||||
logger.warning(
|
||||
"P2PSession %s: lookup response timed out for kv_request_id=%s "
|
||||
"after %.1fs without progress; disconnecting peer session",
|
||||
self.peer_id,
|
||||
req_id,
|
||||
elapsed,
|
||||
)
|
||||
with contextlib.suppress(Exception):
|
||||
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
|
||||
self._conn.mark_dead()
|
||||
|
||||
new_fetch_ids = self._new_fetch_ids
|
||||
self._new_fetch_ids = []
|
||||
return SessionPollResult(
|
||||
|
||||
Reference in New Issue
Block a user