[Bugfix][Core] shm_broadcast: bound idle reader waits and release read slots (#45224)
Signed-off-by: Chaemin Lim <chaemin.lim@mangoboost.io> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Edwin Lim <edwin.lim@mangoboost.io> Co-authored-by: Jaeyoun Kim <jaeyoun.kim@mangoboost.io> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Edwin Lim
Jaeyoun Kim
Nick Hill
parent
ac36a7a1e7
commit
10c75477b0
@@ -348,6 +348,136 @@ def test_message_queue_busy_to_idle():
|
||||
distributed_run(worker_fn_test_busy_to_idle, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("should_warn", [False, True])
|
||||
def test_reader_timeout_caps_indefinite_waits(should_warn):
|
||||
with (
|
||||
mock.patch(
|
||||
"vllm.distributed.device_communicators.shm_broadcast."
|
||||
"SHM_READER_RECHECK_INTERVAL_MS",
|
||||
new=7,
|
||||
),
|
||||
mock.patch(
|
||||
"vllm.distributed.device_communicators.shm_broadcast."
|
||||
"VLLM_RINGBUFFER_WARNING_INTERVAL",
|
||||
new=60,
|
||||
),
|
||||
):
|
||||
timeout = MessageQueue.ReadTimeoutWithWarnings(
|
||||
timeout=None, should_warn=should_warn
|
||||
)
|
||||
assert timeout.timeout_ms() == 7
|
||||
|
||||
|
||||
def test_reader_rechecks_shm_after_idle_wait_timeout_without_notify():
|
||||
writer = MessageQueue(
|
||||
n_reader=1,
|
||||
n_local_reader=1,
|
||||
max_chunk_bytes=1024 * 1024,
|
||||
max_chunks=1,
|
||||
)
|
||||
reader = MessageQueue.create_from_handle(writer.export_handle(), rank=0)
|
||||
payload = 123
|
||||
poll_started = threading.Event()
|
||||
allow_timeout = threading.Event()
|
||||
result = {}
|
||||
|
||||
def acquire_read_in_thread():
|
||||
try:
|
||||
with reader.acquire_read(indefinite=True) as buf:
|
||||
result["value"] = buf[0]
|
||||
except Exception as exc:
|
||||
result["exc"] = exc
|
||||
|
||||
def poll_timeout(*, timeout: int | None = None):
|
||||
poll_started.set()
|
||||
assert allow_timeout.wait(timeout=5)
|
||||
return []
|
||||
|
||||
try:
|
||||
writer.wait_until_ready()
|
||||
reader.wait_until_ready()
|
||||
reader._spin_condition.last_read = 0
|
||||
reader._spin_condition.busy_loop_s = 0
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"vllm.distributed.device_communicators.shm_broadcast."
|
||||
"SHM_READER_RECHECK_INTERVAL_MS",
|
||||
new=50,
|
||||
),
|
||||
mock.patch(
|
||||
"vllm.distributed.device_communicators.shm_broadcast."
|
||||
"VLLM_RINGBUFFER_WARNING_INTERVAL",
|
||||
new=60,
|
||||
),
|
||||
mock.patch.object(
|
||||
reader._spin_condition.poller,
|
||||
"poll",
|
||||
side_effect=poll_timeout,
|
||||
) as poll,
|
||||
):
|
||||
read_thread = threading.Thread(target=acquire_read_in_thread, daemon=True)
|
||||
read_thread.start()
|
||||
assert poll_started.wait(timeout=5)
|
||||
with writer.acquire_write(timeout=0.1) as buf:
|
||||
buf[0] = payload
|
||||
allow_timeout.set()
|
||||
read_thread.join(timeout=5)
|
||||
|
||||
assert not read_thread.is_alive()
|
||||
poll.assert_called_once_with(timeout=50)
|
||||
|
||||
if "exc" in result:
|
||||
raise result["exc"]
|
||||
assert result["value"] == payload
|
||||
with writer.buffer.get_metadata(0) as metadata_buffer:
|
||||
assert metadata_buffer[0] == 1
|
||||
assert metadata_buffer[1] == 1
|
||||
finally:
|
||||
writer.shutdown()
|
||||
reader.shutdown()
|
||||
for socket in (
|
||||
writer.local_socket,
|
||||
writer._spin_condition.local_notify_socket,
|
||||
reader.local_socket,
|
||||
reader._spin_condition.local_notify_socket,
|
||||
reader._spin_condition.read_cancel_socket,
|
||||
reader._spin_condition.write_cancel_socket,
|
||||
):
|
||||
socket.close(linger=0)
|
||||
|
||||
|
||||
def test_acquire_read_releases_slot_when_reader_raises():
|
||||
writer = MessageQueue(
|
||||
n_reader=1,
|
||||
n_local_reader=1,
|
||||
max_chunk_bytes=1024 * 1024,
|
||||
max_chunks=1,
|
||||
)
|
||||
reader = MessageQueue.create_from_handle(writer.export_handle(), rank=0)
|
||||
try:
|
||||
writer.wait_until_ready()
|
||||
reader.wait_until_ready()
|
||||
|
||||
writer.enqueue({"payload": "first"})
|
||||
|
||||
with (
|
||||
pytest.raises(RuntimeError, match="reader failed"),
|
||||
reader.acquire_read(timeout=0.1),
|
||||
):
|
||||
raise RuntimeError("reader failed")
|
||||
|
||||
with writer.buffer.get_metadata(0) as metadata_buffer:
|
||||
assert metadata_buffer[0] == 1
|
||||
assert metadata_buffer[1] == 1
|
||||
|
||||
with writer.acquire_write(timeout=0.1) as buf:
|
||||
buf[0] = 0
|
||||
finally:
|
||||
writer.shutdown()
|
||||
reader.shutdown()
|
||||
|
||||
|
||||
def test_warning_logs(caplog_vllm):
|
||||
"""
|
||||
Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals
|
||||
|
||||
@@ -57,6 +57,11 @@ if TYPE_CHECKING:
|
||||
from _typeshed import SizedBuffer
|
||||
|
||||
VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
|
||||
# Cap on how long an idle reader parks before re-reading the authoritative SHM
|
||||
# written-flag. Bounds lost-notify recovery latency to ~5s while the periodic
|
||||
# wakeup stays negligible (one flag check per reader every 5s).
|
||||
SHM_READER_RECHECK_INTERVAL_MS = 5000
|
||||
|
||||
|
||||
from_bytes_big = functools.partial(int.from_bytes, byteorder="big")
|
||||
|
||||
@@ -631,25 +636,22 @@ class MessageQueue:
|
||||
self.n_warning = 1
|
||||
self.timeout = timeout
|
||||
|
||||
def timeout_ms(self) -> int | None:
|
||||
"""Returns a timeout that is:
|
||||
def timeout_ms(self) -> int:
|
||||
"""Returns a timeout, capped at the recheck interval, that is:
|
||||
- min(time to deadline, time to next warning) if we're logging warnings
|
||||
- time to deadline, if we're not logging warnings
|
||||
- None if the timeout is None and we're not logging warnings
|
||||
- recheck interval if the timeout is None and we're not logging warnings
|
||||
- raise TimeoutError if we are past the deadline
|
||||
"""
|
||||
warning_wait_time = self.warning_wait_time_ms
|
||||
wait_ms = SHM_READER_RECHECK_INTERVAL_MS
|
||||
if self.warning_wait_time_ms is not None:
|
||||
wait_ms = min(wait_ms, self.warning_wait_time_ms)
|
||||
if self.timeout is None:
|
||||
return warning_wait_time
|
||||
|
||||
return wait_ms
|
||||
time_left_ms = int((self.deadline - time.monotonic()) * 1000)
|
||||
if time_left_ms <= 0:
|
||||
raise TimeoutError
|
||||
|
||||
if warning_wait_time and warning_wait_time < time_left_ms:
|
||||
return warning_wait_time
|
||||
|
||||
return time_left_ms
|
||||
return min(wait_ms, time_left_ms)
|
||||
|
||||
def should_warn(self) -> bool:
|
||||
"""Returns true if it's time to log a warning for a timeout that is not
|
||||
@@ -710,18 +712,18 @@ class MessageQueue:
|
||||
# found a block that is not read by this reader
|
||||
# let caller read from the buffer
|
||||
with self.buffer.get_data(self.current_idx) as buf:
|
||||
yield buf
|
||||
|
||||
# caller has read from the buffer
|
||||
# set the read flag
|
||||
metadata_buffer[self.local_reader_rank + 1] = 1
|
||||
# Memory fence ensures the read flag is visible to the writer.
|
||||
# Without this, writer may not see our read completion and
|
||||
# could wait indefinitely for all readers to finish.
|
||||
memory_fence()
|
||||
self.current_idx = (self.current_idx + 1) % self.buffer.max_chunks
|
||||
|
||||
self._spin_condition.record_read()
|
||||
try:
|
||||
yield buf
|
||||
finally:
|
||||
# caller has read from the buffer; set the read flag.
|
||||
metadata_buffer[self.local_reader_rank + 1] = 1
|
||||
# Memory fence ensures the read flag is visible to the writer.
|
||||
# Without this, writer may not see our read completion and
|
||||
# could wait indefinitely for all readers to finish.
|
||||
memory_fence()
|
||||
next_idx = self.current_idx + 1
|
||||
self.current_idx = next_idx % self.buffer.max_chunks
|
||||
self._spin_condition.record_read()
|
||||
break
|
||||
|
||||
def enqueue(self, obj, timeout: float | None = None):
|
||||
|
||||
Reference in New Issue
Block a user