[Bugfix][Core] shm_broadcast: bound idle reader waits and release read slots (#45224)

Signed-off-by: Chaemin Lim <chaemin.lim@mangoboost.io>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Edwin Lim <edwin.lim@mangoboost.io>
Co-authored-by: Jaeyoun Kim <jaeyoun.kim@mangoboost.io>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
chaeminlim-mb
2026-07-23 18:13:29 +08:00
committed by GitHub
co-authored by Edwin Lim Jaeyoun Kim Nick Hill
parent ac36a7a1e7
commit 10c75477b0
2 changed files with 155 additions and 23 deletions
+130
View File
@@ -348,6 +348,136 @@ def test_message_queue_busy_to_idle():
distributed_run(worker_fn_test_busy_to_idle, 4)
@pytest.mark.parametrize("should_warn", [False, True])
def test_reader_timeout_caps_indefinite_waits(should_warn):
with (
mock.patch(
"vllm.distributed.device_communicators.shm_broadcast."
"SHM_READER_RECHECK_INTERVAL_MS",
new=7,
),
mock.patch(
"vllm.distributed.device_communicators.shm_broadcast."
"VLLM_RINGBUFFER_WARNING_INTERVAL",
new=60,
),
):
timeout = MessageQueue.ReadTimeoutWithWarnings(
timeout=None, should_warn=should_warn
)
assert timeout.timeout_ms() == 7
def test_reader_rechecks_shm_after_idle_wait_timeout_without_notify():
writer = MessageQueue(
n_reader=1,
n_local_reader=1,
max_chunk_bytes=1024 * 1024,
max_chunks=1,
)
reader = MessageQueue.create_from_handle(writer.export_handle(), rank=0)
payload = 123
poll_started = threading.Event()
allow_timeout = threading.Event()
result = {}
def acquire_read_in_thread():
try:
with reader.acquire_read(indefinite=True) as buf:
result["value"] = buf[0]
except Exception as exc:
result["exc"] = exc
def poll_timeout(*, timeout: int | None = None):
poll_started.set()
assert allow_timeout.wait(timeout=5)
return []
try:
writer.wait_until_ready()
reader.wait_until_ready()
reader._spin_condition.last_read = 0
reader._spin_condition.busy_loop_s = 0
with (
mock.patch(
"vllm.distributed.device_communicators.shm_broadcast."
"SHM_READER_RECHECK_INTERVAL_MS",
new=50,
),
mock.patch(
"vllm.distributed.device_communicators.shm_broadcast."
"VLLM_RINGBUFFER_WARNING_INTERVAL",
new=60,
),
mock.patch.object(
reader._spin_condition.poller,
"poll",
side_effect=poll_timeout,
) as poll,
):
read_thread = threading.Thread(target=acquire_read_in_thread, daemon=True)
read_thread.start()
assert poll_started.wait(timeout=5)
with writer.acquire_write(timeout=0.1) as buf:
buf[0] = payload
allow_timeout.set()
read_thread.join(timeout=5)
assert not read_thread.is_alive()
poll.assert_called_once_with(timeout=50)
if "exc" in result:
raise result["exc"]
assert result["value"] == payload
with writer.buffer.get_metadata(0) as metadata_buffer:
assert metadata_buffer[0] == 1
assert metadata_buffer[1] == 1
finally:
writer.shutdown()
reader.shutdown()
for socket in (
writer.local_socket,
writer._spin_condition.local_notify_socket,
reader.local_socket,
reader._spin_condition.local_notify_socket,
reader._spin_condition.read_cancel_socket,
reader._spin_condition.write_cancel_socket,
):
socket.close(linger=0)
def test_acquire_read_releases_slot_when_reader_raises():
writer = MessageQueue(
n_reader=1,
n_local_reader=1,
max_chunk_bytes=1024 * 1024,
max_chunks=1,
)
reader = MessageQueue.create_from_handle(writer.export_handle(), rank=0)
try:
writer.wait_until_ready()
reader.wait_until_ready()
writer.enqueue({"payload": "first"})
with (
pytest.raises(RuntimeError, match="reader failed"),
reader.acquire_read(timeout=0.1),
):
raise RuntimeError("reader failed")
with writer.buffer.get_metadata(0) as metadata_buffer:
assert metadata_buffer[0] == 1
assert metadata_buffer[1] == 1
with writer.acquire_write(timeout=0.1) as buf:
buf[0] = 0
finally:
writer.shutdown()
reader.shutdown()
def test_warning_logs(caplog_vllm):
"""
Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals