[Bugfix][KV Offload][P2P] Fix EngineCore crash reconnecting to a reaped peer (#49823)
Signed-off-by: Jason Yao <wsyjh8@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -219,6 +219,7 @@ class TestZmqTransportConnectivity:
|
||||
# Pruning is synchronous within poll().
|
||||
transport_a.poll()
|
||||
assert len(transport_a._connections) == 0
|
||||
assert new_conns[0]._sockets.dealer.closed
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
@@ -228,3 +229,110 @@ class TestZmqTransportConnectivity:
|
||||
transport, _ = _make_transport()
|
||||
transport.close()
|
||||
transport.close() # should not raise
|
||||
|
||||
|
||||
class TestZmqReconnect:
|
||||
"""Reconnecting to a peer whose connection died (real ZMQ sockets).
|
||||
|
||||
A session marks its connection dead while handling messages, which happens
|
||||
after the transport's own sweep has run for that tick — so a dead
|
||||
connection stays registered until the next poll(). Reconnecting in that
|
||||
window must succeed, and the retired connection must release its sockets.
|
||||
Real sockets are required: a mock reports every attribute as closed.
|
||||
"""
|
||||
|
||||
def test_close_after_mark_dead_releases_sockets(self):
|
||||
"""close() releases sockets even when mark_dead() ran first.
|
||||
|
||||
mark_dead() must not set the flag close() guards on, or every peer
|
||||
disconnect leaks a DEALER and a monitor socket.
|
||||
"""
|
||||
transport, _ = _make_transport()
|
||||
try:
|
||||
conn = transport.connect(f"127.0.0.1:{_free_port()}")
|
||||
dealer, monitor = conn._sockets.dealer, conn._sockets.monitor
|
||||
|
||||
conn.mark_dead()
|
||||
assert not conn.alive
|
||||
assert not dealer.closed
|
||||
|
||||
conn.close()
|
||||
assert dealer.closed
|
||||
assert monitor.closed
|
||||
finally:
|
||||
transport.close()
|
||||
|
||||
def test_connect_retires_dead_connection(self):
|
||||
"""connect() replaces a registered-but-dead connection."""
|
||||
transport, _ = _make_transport()
|
||||
try:
|
||||
# The peer port is never bound — only the peer id matters here.
|
||||
peer_id = f"127.0.0.1:{_free_port()}"
|
||||
dead = transport.connect(peer_id)
|
||||
dead.mark_dead()
|
||||
|
||||
conn = transport.connect(peer_id)
|
||||
|
||||
assert conn is not dead
|
||||
assert conn.alive
|
||||
assert transport._connections[peer_id] is conn
|
||||
assert dead._sockets.dealer.closed
|
||||
finally:
|
||||
transport.close()
|
||||
|
||||
def test_repeated_reconnect_to_same_peer(self):
|
||||
"""A flapping peer stays reconnectable.
|
||||
|
||||
Monitor endpoints are inproc addresses that libzmq releases
|
||||
asynchronously, so deriving one from peer_id alone makes each
|
||||
reconnect race the previous teardown and fail with EADDRINUSE.
|
||||
"""
|
||||
transport, _ = _make_transport()
|
||||
try:
|
||||
peer_id = f"127.0.0.1:{_free_port()}"
|
||||
for _ in range(10):
|
||||
conn = transport.connect(peer_id)
|
||||
conn.mark_dead()
|
||||
transport.poll()
|
||||
assert conn._sockets.dealer.closed
|
||||
|
||||
assert not transport._connections
|
||||
finally:
|
||||
transport.close()
|
||||
|
||||
def test_inbound_message_survives_dead_registration(self):
|
||||
"""A reconnecting peer's first message is not dropped.
|
||||
|
||||
poll() must retire connections killed by their session before routing
|
||||
traffic, otherwise the message is enqueued into the dead connection and
|
||||
discarded when it is swept — and a session announces itself only once.
|
||||
Covers only that between-polls window: a peer dying while poll() runs,
|
||||
or dying silently until the heartbeat expires, is out of scope.
|
||||
"""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, _ = _make_transport()
|
||||
|
||||
try:
|
||||
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
conn_b.send({"type": "connect", "seq": 1})
|
||||
|
||||
inbound = _wait_for_inbound(transport_a)[0]
|
||||
_wait_for_messages(transport_a, inbound, 1)
|
||||
|
||||
inbound.mark_dead()
|
||||
conn_b.send({"type": "connect", "seq": 2})
|
||||
|
||||
# Wait until the frame is readable on the ROUTER, so the message is
|
||||
# known to have arrived rather than merely being slow.
|
||||
poller = zmq.Poller()
|
||||
poller.register(transport_a._router, zmq.POLLIN)
|
||||
assert poller.poll(2000), "message never reached the ROUTER"
|
||||
|
||||
new_conns = _wait_for_inbound(transport_a)
|
||||
assert len(new_conns) == 1
|
||||
assert new_conns[0] is not inbound
|
||||
msgs = _wait_for_messages(transport_a, new_conns[0], 1)
|
||||
assert msgs == [{"type": "connect", "seq": 2}]
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
Reference in New Issue
Block a user