diff --git a/tests/distributed/test_shm_broadcast.py b/tests/distributed/test_shm_broadcast.py index 7cf3b01e75..33affb2a39 100644 --- a/tests/distributed/test_shm_broadcast.py +++ b/tests/distributed/test_shm_broadcast.py @@ -348,6 +348,136 @@ def test_message_queue_busy_to_idle(): distributed_run(worker_fn_test_busy_to_idle, 4) +@pytest.mark.parametrize("should_warn", [False, True]) +def test_reader_timeout_caps_indefinite_waits(should_warn): + with ( + mock.patch( + "vllm.distributed.device_communicators.shm_broadcast." + "SHM_READER_RECHECK_INTERVAL_MS", + new=7, + ), + mock.patch( + "vllm.distributed.device_communicators.shm_broadcast." + "VLLM_RINGBUFFER_WARNING_INTERVAL", + new=60, + ), + ): + timeout = MessageQueue.ReadTimeoutWithWarnings( + timeout=None, should_warn=should_warn + ) + assert timeout.timeout_ms() == 7 + + +def test_reader_rechecks_shm_after_idle_wait_timeout_without_notify(): + writer = MessageQueue( + n_reader=1, + n_local_reader=1, + max_chunk_bytes=1024 * 1024, + max_chunks=1, + ) + reader = MessageQueue.create_from_handle(writer.export_handle(), rank=0) + payload = 123 + poll_started = threading.Event() + allow_timeout = threading.Event() + result = {} + + def acquire_read_in_thread(): + try: + with reader.acquire_read(indefinite=True) as buf: + result["value"] = buf[0] + except Exception as exc: + result["exc"] = exc + + def poll_timeout(*, timeout: int | None = None): + poll_started.set() + assert allow_timeout.wait(timeout=5) + return [] + + try: + writer.wait_until_ready() + reader.wait_until_ready() + reader._spin_condition.last_read = 0 + reader._spin_condition.busy_loop_s = 0 + + with ( + mock.patch( + "vllm.distributed.device_communicators.shm_broadcast." + "SHM_READER_RECHECK_INTERVAL_MS", + new=50, + ), + mock.patch( + "vllm.distributed.device_communicators.shm_broadcast." + "VLLM_RINGBUFFER_WARNING_INTERVAL", + new=60, + ), + mock.patch.object( + reader._spin_condition.poller, + "poll", + side_effect=poll_timeout, + ) as poll, + ): + read_thread = threading.Thread(target=acquire_read_in_thread, daemon=True) + read_thread.start() + assert poll_started.wait(timeout=5) + with writer.acquire_write(timeout=0.1) as buf: + buf[0] = payload + allow_timeout.set() + read_thread.join(timeout=5) + + assert not read_thread.is_alive() + poll.assert_called_once_with(timeout=50) + + if "exc" in result: + raise result["exc"] + assert result["value"] == payload + with writer.buffer.get_metadata(0) as metadata_buffer: + assert metadata_buffer[0] == 1 + assert metadata_buffer[1] == 1 + finally: + writer.shutdown() + reader.shutdown() + for socket in ( + writer.local_socket, + writer._spin_condition.local_notify_socket, + reader.local_socket, + reader._spin_condition.local_notify_socket, + reader._spin_condition.read_cancel_socket, + reader._spin_condition.write_cancel_socket, + ): + socket.close(linger=0) + + +def test_acquire_read_releases_slot_when_reader_raises(): + writer = MessageQueue( + n_reader=1, + n_local_reader=1, + max_chunk_bytes=1024 * 1024, + max_chunks=1, + ) + reader = MessageQueue.create_from_handle(writer.export_handle(), rank=0) + try: + writer.wait_until_ready() + reader.wait_until_ready() + + writer.enqueue({"payload": "first"}) + + with ( + pytest.raises(RuntimeError, match="reader failed"), + reader.acquire_read(timeout=0.1), + ): + raise RuntimeError("reader failed") + + with writer.buffer.get_metadata(0) as metadata_buffer: + assert metadata_buffer[0] == 1 + assert metadata_buffer[1] == 1 + + with writer.acquire_write(timeout=0.1) as buf: + buf[0] = 0 + finally: + writer.shutdown() + reader.shutdown() + + def test_warning_logs(caplog_vllm): """ Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals diff --git a/vllm/distributed/device_communicators/shm_broadcast.py b/vllm/distributed/device_communicators/shm_broadcast.py index 43e066c44b..6b9dd4068b 100644 --- a/vllm/distributed/device_communicators/shm_broadcast.py +++ b/vllm/distributed/device_communicators/shm_broadcast.py @@ -57,6 +57,11 @@ if TYPE_CHECKING: from _typeshed import SizedBuffer VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL +# Cap on how long an idle reader parks before re-reading the authoritative SHM +# written-flag. Bounds lost-notify recovery latency to ~5s while the periodic +# wakeup stays negligible (one flag check per reader every 5s). +SHM_READER_RECHECK_INTERVAL_MS = 5000 + from_bytes_big = functools.partial(int.from_bytes, byteorder="big") @@ -631,25 +636,22 @@ class MessageQueue: self.n_warning = 1 self.timeout = timeout - def timeout_ms(self) -> int | None: - """Returns a timeout that is: + def timeout_ms(self) -> int: + """Returns a timeout, capped at the recheck interval, that is: - min(time to deadline, time to next warning) if we're logging warnings - time to deadline, if we're not logging warnings - - None if the timeout is None and we're not logging warnings + - recheck interval if the timeout is None and we're not logging warnings - raise TimeoutError if we are past the deadline """ - warning_wait_time = self.warning_wait_time_ms + wait_ms = SHM_READER_RECHECK_INTERVAL_MS + if self.warning_wait_time_ms is not None: + wait_ms = min(wait_ms, self.warning_wait_time_ms) if self.timeout is None: - return warning_wait_time - + return wait_ms time_left_ms = int((self.deadline - time.monotonic()) * 1000) if time_left_ms <= 0: raise TimeoutError - - if warning_wait_time and warning_wait_time < time_left_ms: - return warning_wait_time - - return time_left_ms + return min(wait_ms, time_left_ms) def should_warn(self) -> bool: """Returns true if it's time to log a warning for a timeout that is not @@ -710,18 +712,18 @@ class MessageQueue: # found a block that is not read by this reader # let caller read from the buffer with self.buffer.get_data(self.current_idx) as buf: - yield buf - - # caller has read from the buffer - # set the read flag - metadata_buffer[self.local_reader_rank + 1] = 1 - # Memory fence ensures the read flag is visible to the writer. - # Without this, writer may not see our read completion and - # could wait indefinitely for all readers to finish. - memory_fence() - self.current_idx = (self.current_idx + 1) % self.buffer.max_chunks - - self._spin_condition.record_read() + try: + yield buf + finally: + # caller has read from the buffer; set the read flag. + metadata_buffer[self.local_reader_rank + 1] = 1 + # Memory fence ensures the read flag is visible to the writer. + # Without this, writer may not see our read completion and + # could wait indefinitely for all readers to finish. + memory_fence() + next_idx = self.current_idx + 1 + self.current_idx = next_idx % self.buffer.max_chunks + self._spin_condition.record_read() break def enqueue(self, obj, timeout: float | None = None):