[Bugfix][KV Offload] Bound unanswered P2P lookup waits (#56836)
Signed-off-by: jacklin78911-collab <jacklin78911@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Codex
Robert Shaw
Itay Etelis
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Or Ozeri
Kevin H. Luu
parent
81e1c3f183
commit
3f1bcfd82b
@@ -0,0 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.v1.kv_offload.tiering.p2p.session import client as client_module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def lookup_clock(monkeypatch):
|
||||
"""Advance monotonic time for P2P deadlines without sleeping."""
|
||||
clock = SimpleNamespace(now=0.0)
|
||||
monkeypatch.setattr(client_module.time, "monotonic", lambda: clock.now)
|
||||
return clock
|
||||
@@ -37,6 +37,7 @@ from vllm.v1.kv_offload.tiering.p2p.session import (
|
||||
SessionPollResult,
|
||||
StoreResult,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.client import _LOOKUP_TIMEOUT_S
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -1172,6 +1173,108 @@ def _build_paired_managers() -> tuple[P2PSecondaryTierManager, P2PSecondaryTierM
|
||||
class TestBidirectionalManager:
|
||||
"""Two managers each load FROM and serve TO the other over a single peer."""
|
||||
|
||||
def test_unanswered_lookup_falls_back_to_local_prefill(self, lookup_clock):
|
||||
mgr_a, mgr_b = _build_paired_managers()
|
||||
ctx = _req_context(_remote_kv_source_kv_params("B", 2))
|
||||
mgr_a.on_new_request(ctx)
|
||||
for _ in range(3):
|
||||
list(mgr_a.get_finished_jobs())
|
||||
list(mgr_b.get_finished_jobs())
|
||||
assert mgr_a._sessions["B:2"].ready
|
||||
assert mgr_a.lookup(b"key", ctx) == LookupResult.RETRY
|
||||
mgr_a.on_schedule_end(ScheduleEndContext(new_req_ids=[], preempted_req_ids=[]))
|
||||
|
||||
# Stop the server's progress without closing its live connection.
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
|
||||
assert list(mgr_a.get_finished_jobs()) == []
|
||||
assert mgr_a.lookup(b"key", ctx) == LookupResult.RETRY
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S
|
||||
assert list(mgr_a.get_finished_jobs()) == []
|
||||
assert mgr_a.lookup(b"key", ctx) == LookupResult.MISS
|
||||
assert "B:2" not in mgr_a._sessions
|
||||
assert "B:2" not in mgr_a._data._remote_peers
|
||||
|
||||
# A replacement session must not restart the expired request's probes.
|
||||
mgr_a._sessions["B:2"] = _FakeSession(peer_id="B:2") # type: ignore[assignment]
|
||||
assert mgr_a.lookup(b"key", ctx) == LookupResult.MISS
|
||||
mgr_a.on_request_finished(ctx)
|
||||
assert "req-1" not in mgr_a._failed_req_ids
|
||||
|
||||
def test_lookup_expiry_allows_zmq_reconnect(self, lookup_clock):
|
||||
"""The peer must retire the timed-out session before a new handshake."""
|
||||
import zmq
|
||||
from zmq.utils.monitor import recv_monitor_message
|
||||
|
||||
from .test_zmq_transport import _make_transport
|
||||
|
||||
managers = []
|
||||
monitors = []
|
||||
try:
|
||||
for _ in range(2):
|
||||
mgr = _make_manager()
|
||||
mgr._control, port = _make_transport()
|
||||
mgr._local_id = f"127.0.0.1:{port}"
|
||||
mgr._data = _FakeData(mgr._local_id)
|
||||
managers.append(mgr)
|
||||
monitors.append(
|
||||
mgr._control._router.get_monitor_socket(zmq.EVENT_DISCONNECTED)
|
||||
)
|
||||
mgr_a, mgr_b = managers
|
||||
peer_a, peer_b = mgr_a._local_id, mgr_b._local_id
|
||||
port_b = int(peer_b.rsplit(":", 1)[1])
|
||||
|
||||
def poll_until(condition):
|
||||
deadline = time.perf_counter() + 5.0
|
||||
while time.perf_counter() < deadline:
|
||||
list(mgr_a.get_finished_jobs())
|
||||
list(mgr_b.get_finished_jobs())
|
||||
if condition():
|
||||
return
|
||||
time.sleep(0.005)
|
||||
pytest.fail("P2P session did not reach the expected state")
|
||||
|
||||
def both_ready():
|
||||
return (
|
||||
peer_b in mgr_a._sessions
|
||||
and peer_a in mgr_b._sessions
|
||||
and mgr_a._sessions[peer_b].ready
|
||||
and mgr_b._sessions[peer_a].ready
|
||||
)
|
||||
|
||||
ctx = _req_context(_remote_kv_source_kv_params("127.0.0.1", port_b))
|
||||
mgr_a.on_new_request(ctx)
|
||||
poll_until(both_ready)
|
||||
old_peer_session = mgr_b._sessions[peer_a]
|
||||
assert mgr_a.lookup(b"key", ctx) == LookupResult.RETRY
|
||||
mgr_a.on_schedule_end(
|
||||
ScheduleEndContext(new_req_ids=[], preempted_req_ids=[])
|
||||
)
|
||||
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S
|
||||
list(mgr_a.get_finished_jobs())
|
||||
assert mgr_a.lookup(b"key", ctx) == LookupResult.MISS
|
||||
assert peer_b not in mgr_a._sessions
|
||||
poll_until(lambda: peer_a not in mgr_b._sessions)
|
||||
# Wait for old TCP identities to retire before reconnecting.
|
||||
for monitor in monitors:
|
||||
assert monitor.poll(5000), "old TCP connection did not disconnect"
|
||||
assert recv_monitor_message(monitor)["event"] == zmq.EVENT_DISCONNECTED
|
||||
|
||||
new_ctx = _req_context(
|
||||
_remote_kv_source_kv_params("127.0.0.1", port_b, "req-2")
|
||||
)
|
||||
mgr_a.on_new_request(new_ctx)
|
||||
poll_until(both_ready)
|
||||
assert mgr_b._sessions[peer_a] is not old_peer_session
|
||||
assert mgr_a.lookup(b"key", ctx) == LookupResult.MISS
|
||||
finally:
|
||||
for monitor in monitors:
|
||||
monitor.close(linger=0)
|
||||
# Avoid queuing shutdown messages to an already closed peer.
|
||||
for mgr in managers:
|
||||
mgr._control.close()
|
||||
mgr.shutdown()
|
||||
|
||||
def test_both_loads_succeed(self):
|
||||
mgr_a, mgr_b = _build_paired_managers()
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.v1.kv_offload.tiering.p2p.session import (
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.client import (
|
||||
_ABORT_ACK_TIMEOUT_S,
|
||||
_LOAD_TIMEOUT_S,
|
||||
_LOOKUP_TIMEOUT_S,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
|
||||
TYPE_KEY,
|
||||
@@ -839,6 +840,215 @@ class TestClientFlows:
|
||||
class TestLookupFlow:
|
||||
"""Consumer-side state machine for do_p2p_fetch lookups."""
|
||||
|
||||
def test_lookup_deadline_starts_after_handshake(self, lookup_clock):
|
||||
session, conn, _ = _make_session()
|
||||
session.register_lookup("req-1", b"hA")
|
||||
session.flush_pending_lookups()
|
||||
# Complete the handshake before its independent timeout expires.
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
|
||||
session.poll()
|
||||
assert session.alive
|
||||
|
||||
_activate(session, conn)
|
||||
session.flush_pending_lookups()
|
||||
lookups = [m for m in conn._sent if m[TYPE_KEY] == LookupMsg.TYPE]
|
||||
assert len(lookups) == 1
|
||||
lookup_clock.now += _LOOKUP_TIMEOUT_S - 1
|
||||
session.poll()
|
||||
assert session.alive
|
||||
lookup_clock.now += 1
|
||||
session.poll()
|
||||
assert not session.alive
|
||||
assert session.close().failed_req_ids == ["req-1"]
|
||||
|
||||
def test_partial_response_rearms_lookup_deadline(self, lookup_clock):
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
session.register_lookup("req-1", b"hA")
|
||||
session.register_lookup("req-1", b"hB")
|
||||
session.flush_pending_lookups()
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: LookupRespMsg.TYPE,
|
||||
LookupRespMsg.KV_REQUEST_ID: "req-1",
|
||||
LookupRespMsg.KEYS: [b"hA"],
|
||||
LookupRespMsg.HITS: [True],
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
assert session.register_lookup("req-1", b"hA") is True
|
||||
assert session.register_lookup("req-1", b"hB") is None
|
||||
session.register_lookup("req-1", b"hC")
|
||||
session.flush_pending_lookups()
|
||||
|
||||
lookup_clock.now += 1
|
||||
session.poll()
|
||||
assert session.alive
|
||||
lookup_clock.now += _LOOKUP_TIMEOUT_S - 1
|
||||
session.poll()
|
||||
assert not session.alive
|
||||
assert session.close().failed_req_ids == ["req-1"]
|
||||
|
||||
def test_lookup_progress_keeps_overlapping_batches_alive(self, lookup_clock):
|
||||
"""Prompt batch replies keep a long-running lookup phase alive."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
previous_key = b"key-0"
|
||||
session.register_lookup("req-1", previous_key)
|
||||
session.flush_pending_lookups()
|
||||
|
||||
for step in range(1, int(2 * _LOOKUP_TIMEOUT_S) + 1):
|
||||
lookup_clock.now = float(step)
|
||||
next_key = f"key-{step}".encode()
|
||||
hit = step % 2 == 0
|
||||
session.register_lookup("req-1", next_key)
|
||||
session.flush_pending_lookups()
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: LookupRespMsg.TYPE,
|
||||
LookupRespMsg.KV_REQUEST_ID: "req-1",
|
||||
LookupRespMsg.KEYS: [previous_key],
|
||||
LookupRespMsg.HITS: [hit],
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
assert session.alive
|
||||
assert session.register_lookup("req-1", previous_key) is hit
|
||||
previous_key = next_key
|
||||
|
||||
def test_buffered_lookup_response_precedes_expiry(self, lookup_clock):
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
session.register_lookup("req-1", b"hA")
|
||||
session.flush_pending_lookups()
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: LookupRespMsg.TYPE,
|
||||
LookupRespMsg.KV_REQUEST_ID: "req-1",
|
||||
LookupRespMsg.KEYS: [b"hA"],
|
||||
LookupRespMsg.HITS: [True],
|
||||
}
|
||||
)
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S
|
||||
session.poll()
|
||||
assert session.alive
|
||||
assert session.register_lookup("req-1", b"hA") is True
|
||||
|
||||
def test_duplicate_responses_and_new_keys_do_not_rearm(self, lookup_clock):
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
session.register_lookup("req-1", b"hA")
|
||||
session.register_lookup("req-1", b"hB")
|
||||
session.flush_pending_lookups()
|
||||
response = {
|
||||
TYPE_KEY: LookupRespMsg.TYPE,
|
||||
LookupRespMsg.KV_REQUEST_ID: "req-1",
|
||||
LookupRespMsg.KEYS: [b"hA", b"unknown"],
|
||||
LookupRespMsg.HITS: [True, False],
|
||||
}
|
||||
conn.enqueue(response)
|
||||
session.poll()
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
|
||||
conn.enqueue(response)
|
||||
session.register_lookup("req-1", b"hC")
|
||||
session.flush_pending_lookups()
|
||||
session.poll()
|
||||
assert session.alive
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S
|
||||
session.poll()
|
||||
assert not session.alive
|
||||
|
||||
@pytest.mark.parametrize("partial_fetch", [False, True])
|
||||
def test_lookup_expiry_preserves_completed_transfers(
|
||||
self, lookup_clock, partial_fetch
|
||||
):
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
session.register_lookup("stalled", b"key")
|
||||
session.flush_pending_lookups()
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
|
||||
session.request_blocks(1, "load", [b"key"], [0])
|
||||
session.add_stored_blocks("store", [b"key"], [1], 2)
|
||||
if partial_fetch:
|
||||
session.add_stored_blocks("store", [b"unused"], [2], 3)
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.KV_REQUEST_ID: "store",
|
||||
FetchMsg.KEYS: [b"key"],
|
||||
FetchMsg.BLOCK_INDEXES: [0],
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
transport._poll_done = list(transport._transfers)
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.KV_REQUEST_ID: "load",
|
||||
TransferDoneMsg.SUCCESS: True,
|
||||
TransferDoneMsg.ROUND_SEQ: 0,
|
||||
}
|
||||
)
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S
|
||||
result = session.poll()
|
||||
assert not session.alive
|
||||
assert result.loads == [LoadResult(1, "load", True)]
|
||||
expected_stores = [StoreResult(2, True)]
|
||||
if partial_fetch:
|
||||
expected_stores.append(StoreResult(3, True))
|
||||
assert result.stores == expected_stores
|
||||
closed = session.close()
|
||||
assert closed.failed_req_ids == ["stalled"]
|
||||
assert closed.failed_jobs == []
|
||||
assert closed.failed_stores == []
|
||||
|
||||
@pytest.mark.parametrize("action", ["resolve", "finish", "fetch"])
|
||||
def test_lookup_deadline_cleared_before_next_batch(self, lookup_clock, action):
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
session.register_lookup("req-1", b"hA")
|
||||
session.register_lookup("req-1", b"hB")
|
||||
session.flush_pending_lookups()
|
||||
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: LookupRespMsg.TYPE,
|
||||
LookupRespMsg.KV_REQUEST_ID: "req-1",
|
||||
LookupRespMsg.KEYS: [b"hA"],
|
||||
LookupRespMsg.HITS: [True],
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
if action == "resolve":
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: LookupRespMsg.TYPE,
|
||||
LookupRespMsg.KV_REQUEST_ID: "req-1",
|
||||
LookupRespMsg.KEYS: [b"hB"],
|
||||
LookupRespMsg.HITS: [False],
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
elif action == "finish":
|
||||
session.finish_request("req-1")
|
||||
else:
|
||||
session.request_blocks(1, "req-1", [b"hA"], [0])
|
||||
|
||||
lookup_clock.now += 2
|
||||
session.poll()
|
||||
assert session.alive
|
||||
session.register_lookup("req-1", b"hC")
|
||||
session.flush_pending_lookups()
|
||||
lookup_clock.now += _LOOKUP_TIMEOUT_S - 1
|
||||
session.poll()
|
||||
assert session.alive
|
||||
lookup_clock.now += 1
|
||||
session.poll()
|
||||
assert not session.alive
|
||||
|
||||
def test_aggregate_flush_resolve_round_trip(self):
|
||||
"""register_lookup → flush sends one LookupMsg → response
|
||||
resolves entries → register_lookup returns the cached bool on
|
||||
@@ -2531,20 +2741,19 @@ class TestFinishRequestServerSide:
|
||||
|
||||
# Transfer 1 completes — now ``_has_inflight_for("req-1")`` is False
|
||||
# and the elif branch in collect_results fires _finalize(success=False).
|
||||
# The k1 success result is direct; the k2 failure result is queued
|
||||
# in _pending_store_results and surfaces on the NEXT poll.
|
||||
# Both the k1 success and the deferred k2 failure surface in this poll.
|
||||
transport._poll_done.append(tid_1)
|
||||
stores_first = session.poll().stores
|
||||
assert StoreResult(job_id=100, success=True) in stores_first
|
||||
assert session.poll().stores == [
|
||||
StoreResult(job_id=100, success=True),
|
||||
StoreResult(job_id=200, success=False),
|
||||
]
|
||||
# Outbound state cleaned up; peer notified with success=False.
|
||||
assert _srv_outbound(session, "req-1") is None
|
||||
done = next(m for m in conn._sent if m.get(TYPE_KEY) == TransferDoneMsg.TYPE)
|
||||
assert done[TransferDoneMsg.KV_REQUEST_ID] == "req-1"
|
||||
assert done[TransferDoneMsg.SUCCESS] is False
|
||||
|
||||
# Next poll drains the queued failure.
|
||||
stores_second = session.poll().stores
|
||||
assert StoreResult(job_id=200, success=False) in stores_second
|
||||
assert session.poll().stores == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user