[Bugfix] Don't reuse engine core payload buffer while zmq is sending it (#50053)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Nick Hill
2026-07-28 16:09:28 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent 4fb483ca86
commit 6453fc0b8c
4 changed files with 179 additions and 58 deletions
+12 -46
View File
@@ -7,7 +7,7 @@ import sys
import uuid
import weakref
from abc import ABC, abstractmethod
from collections import Counter, defaultdict, deque
from collections import Counter, defaultdict
from collections.abc import Awaitable, Callable, Sequence
from concurrent.futures import Future
from dataclasses import dataclass
@@ -671,11 +671,6 @@ class MPClient(EngineCoreClient):
self.core_engine: EngineIdentity = self.core_engines[0]
self.utility_results: dict[int, AnyFuture] = {}
# Request objects which may contain pytorch-allocated tensors
# that we need to keep references to until zmq is done with the
# underlying data.
self.pending_messages = deque[tuple[zmq.MessageTracker, Any]]()
# Start monitoring engine core processes for unexpected failures
self.start_engine_core_monitor()
@@ -707,14 +702,6 @@ class MPClient(EngineCoreClient):
if self.resources.engine_dead:
raise EngineDeadError()
def add_pending_message(self, tracker: zmq.MessageTracker, msg: Any):
if not tracker.done:
self.pending_messages.appendleft((tracker, msg))
def free_pending_messages(self):
while self.pending_messages and self.pending_messages[-1][0].done:
self.pending_messages.pop()
def dp_engines_running(self) -> bool:
return self.engines_running
@@ -896,17 +883,12 @@ class SyncMPClient(MPClient):
def _send_input(self, request_type: EngineCoreRequestType, request: Any):
self.ensure_alive()
self.free_pending_messages()
# (Identity, RequestType, SerializedRequest)
msg = (self.core_engine, request_type.value, *self.encoder.encode(request))
if len(msg) <= 3:
# No auxiliary buffers => no tensor backing buffers in request.
self.input_socket.send_multipart(msg, copy=False)
return
tracker = self.input_socket.send_multipart(msg, copy=False, track=True)
self.add_pending_message(tracker, request)
# Any zero-copy tensor/ndarray frames are kept alive by zmq itself
# until it's finished sending them (there is a ref chain from the underlying
# memoryview back to the original owning tensor/ndarray).
self.input_socket.send_multipart(msg, copy=False)
def call_utility(self, method: str, *args) -> Any:
call_id = uuid.uuid1().int >> 64
@@ -1129,32 +1111,16 @@ class AsyncMPClient(MPClient):
engine = self.core_engine
message = (request_type.value, *self.encoder.encode(request))
return self._send_input_message(message, engine, request)
return self._send_input_message(message, engine)
def _send_input_message(
self, message: tuple[bytestr, ...], engine: EngineIdentity, objects: Any
self, message: tuple[bytestr, ...], engine: EngineIdentity
) -> Awaitable[Any]:
"""
objects is a reference to retain until zmq is finished with the
buffers, in case they were extracted from tensors in the request.
"""
self.ensure_alive()
self.free_pending_messages()
msg = (engine,) + message
if not objects or len(msg) <= 3:
# No auxiliary buffers => no tensor backing buffers in request.
return self.input_socket.send_multipart(msg, copy=False)
future: asyncio.Future[zmq.MessageTracker]
future = self.input_socket.send_multipart(msg, copy=False, track=True)
def add_pending(f: asyncio.Future[zmq.MessageTracker]):
with contextlib.suppress(BaseException):
self.add_pending_message(f.result(), objects)
future.add_done_callback(add_pending)
return future
# Any zero-copy tensor/ndarray frames are kept alive by zmq itself
# until it's finished sending them (there is a ref chain from the underlying
# memoryview back to the original owning tensor/ndarray).
return self.input_socket.send_multipart((engine,) + message, copy=False)
async def call_utility_async(self, method: str, *args) -> Any:
return await self._call_utility_async(method, *args, engine=self.core_engine)
@@ -1169,7 +1135,7 @@ class AsyncMPClient(MPClient):
EngineCoreRequestType.UTILITY.value,
*self.encoder.encode((self.client_index, call_id, method, args)),
)
await self._send_input_message(message, engine, args)
await self._send_input_message(message, engine)
self._ensure_output_queue_task()
return await future