[Bugfix] Don't reuse engine core payload buffer while zmq is sending it (#50053)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Nick Hill
2026-07-28 16:09:28 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent 4fb483ca86
commit 6453fc0b8c
4 changed files with 179 additions and 58 deletions
+30 -11
View File
@@ -7,7 +7,7 @@ import signal
import threading
import time
from collections import defaultdict, deque
from collections.abc import Callable, Generator
from collections.abc import Callable, Generator, Sequence
from concurrent.futures import Future
from contextlib import ExitStack, contextmanager
from enum import IntEnum
@@ -87,7 +87,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig, get_kv_cache_spec_kind
from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder, bytestr
from vllm.v1.structured_output import StructuredOutputManager
from vllm.v1.utils import compute_iteration_details
from vllm.version import __version__ as VLLM_VERSION
@@ -1748,10 +1748,11 @@ class EngineCoreProc(EngineCore):
encoder = MsgpackEncoder()
# Send buffers to reuse.
reuse_buffers: list[bytearray] = []
# Keep references to outputs and buffers until zmq is finished
# with them (outputs may contain tensors/np arrays whose
# backing buffers were extracted for zero-copy send).
pending = deque[tuple[zmq.MessageTracker, Any, bytearray]]()
# Payload buffers that can't be reused yet because zmq may still be
# sending them.
# Buffers of the zero-copy tensor/ndarray frames don't need tracking
# here: zmq itself holds a reference to each until it's done with it.
pending = deque[tuple[zmq.MessageTracker, bytearray]]()
# We must set linger to ensure the ENGINE_CORE_DEAD
# message is sent prior to closing the socket.
@@ -1792,20 +1793,38 @@ class EngineCoreProc(EngineCore):
# Reclaim buffers that zmq is finished with.
while pending and pending[-1][0].done:
reuse_buffers.append(pending.pop()[2])
reclaimed = pending.pop()[1]
if len(reuse_buffers) < max_reuse_bufs:
reuse_buffers.append(reclaimed)
buffer = reuse_buffers.pop() if reuse_buffers else bytearray()
buffers = encoder.encode_into(outputs, buffer)
tracker = sockets[client_index].send_multipart(
buffers, copy=False, track=True
tracker = self._send_msg_tracking_payload(
sockets[client_index], buffers
)
if not tracker.done:
ref = outputs if len(buffers) > 1 else None
pending.appendleft((tracker, ref, buffer))
pending.appendleft((tracker, buffer))
elif len(reuse_buffers) < max_reuse_bufs:
# Limit the number of buffers to reuse.
reuse_buffers.append(buffer)
@staticmethod
def _send_msg_tracking_payload(
socket: zmq.Socket, buffers: Sequence[bytestr]
) -> zmq.MessageTracker:
"""Send `buffers` as a zero-copy multipart message, returning a tracker
for the *first* frame.
Used instead of `Socket.send_multipart()` because we reuse the buffer
passed to `MsgpackEncoder.encode_into()`: `send_multipart()` returns a
tracker for the last frame only.
"""
more_flag = zmq.SNDMORE if len(buffers) > 1 else 0
tracker = socket.send(buffers[0], more_flag, copy=False, track=True)
if more_flag:
socket.send_multipart(buffers[1:], copy=False)
return tracker
def _handle_request_preproc_error(self, request: EngineCoreRequest) -> None:
"""Log and return a request-scoped error response for exceptions raised
from the add request preprocessing in the input socket processing thread.