[Bugfix][KV Offload][P2P] Fix EngineCore crash reconnecting to a reaped peer (#49823)

Signed-off-by: Jason Yao <wsyjh8@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Jason
2026-07-27 08:07:00 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 49f31d7cee
commit 53397fbfac
3 changed files with 167 additions and 13 deletions
@@ -219,6 +219,7 @@ class TestZmqTransportConnectivity:
# Pruning is synchronous within poll().
transport_a.poll()
assert len(transport_a._connections) == 0
assert new_conns[0]._sockets.dealer.closed
finally:
transport_a.close()
transport_b.close()
@@ -228,3 +229,110 @@ class TestZmqTransportConnectivity:
transport, _ = _make_transport()
transport.close()
transport.close() # should not raise
class TestZmqReconnect:
"""Reconnecting to a peer whose connection died (real ZMQ sockets).
A session marks its connection dead while handling messages, which happens
after the transport's own sweep has run for that tick — so a dead
connection stays registered until the next poll(). Reconnecting in that
window must succeed, and the retired connection must release its sockets.
Real sockets are required: a mock reports every attribute as closed.
"""
def test_close_after_mark_dead_releases_sockets(self):
"""close() releases sockets even when mark_dead() ran first.
mark_dead() must not set the flag close() guards on, or every peer
disconnect leaks a DEALER and a monitor socket.
"""
transport, _ = _make_transport()
try:
conn = transport.connect(f"127.0.0.1:{_free_port()}")
dealer, monitor = conn._sockets.dealer, conn._sockets.monitor
conn.mark_dead()
assert not conn.alive
assert not dealer.closed
conn.close()
assert dealer.closed
assert monitor.closed
finally:
transport.close()
def test_connect_retires_dead_connection(self):
"""connect() replaces a registered-but-dead connection."""
transport, _ = _make_transport()
try:
# The peer port is never bound — only the peer id matters here.
peer_id = f"127.0.0.1:{_free_port()}"
dead = transport.connect(peer_id)
dead.mark_dead()
conn = transport.connect(peer_id)
assert conn is not dead
assert conn.alive
assert transport._connections[peer_id] is conn
assert dead._sockets.dealer.closed
finally:
transport.close()
def test_repeated_reconnect_to_same_peer(self):
"""A flapping peer stays reconnectable.
Monitor endpoints are inproc addresses that libzmq releases
asynchronously, so deriving one from peer_id alone makes each
reconnect race the previous teardown and fail with EADDRINUSE.
"""
transport, _ = _make_transport()
try:
peer_id = f"127.0.0.1:{_free_port()}"
for _ in range(10):
conn = transport.connect(peer_id)
conn.mark_dead()
transport.poll()
assert conn._sockets.dealer.closed
assert not transport._connections
finally:
transport.close()
def test_inbound_message_survives_dead_registration(self):
"""A reconnecting peer's first message is not dropped.
poll() must retire connections killed by their session before routing
traffic, otherwise the message is enqueued into the dead connection and
discarded when it is swept — and a session announces itself only once.
Covers only that between-polls window: a peer dying while poll() runs,
or dying silently until the heartbeat expires, is out of scope.
"""
transport_a, port_a = _make_transport()
transport_b, _ = _make_transport()
try:
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
conn_b.send({"type": "connect", "seq": 1})
inbound = _wait_for_inbound(transport_a)[0]
_wait_for_messages(transport_a, inbound, 1)
inbound.mark_dead()
conn_b.send({"type": "connect", "seq": 2})
# Wait until the frame is readable on the ROUTER, so the message is
# known to have arrived rather than merely being slow.
poller = zmq.Poller()
poller.register(transport_a._router, zmq.POLLIN)
assert poller.poll(2000), "message never reached the ROUTER"
new_conns = _wait_for_inbound(transport_a)
assert len(new_conns) == 1
assert new_conns[0] is not inbound
msgs = _wait_for_messages(transport_a, new_conns[0], 1)
assert msgs == [{"type": "connect", "seq": 2}]
finally:
transport_a.close()
transport_b.close()