[Bugfix] Don't reuse engine core payload buffer while zmq is sending it (#50053)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Nick Hill
2026-07-28 16:09:28 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent 4fb483ca86
commit 6453fc0b8c
4 changed files with 179 additions and 58 deletions
+136
View File
@@ -423,3 +423,139 @@ def test_multiple_senders_single_receiver_ipc():
assert torch.allclose(decoded.prompt_embeds, original_tensor), (
f"Value mismatch for sender {sender_idx} msg {msg_idx}"
)
def _logprobs_outputs(num_reqs: int, num_prompt_tokens: int):
"""An EngineCoreOutputs carrying prompt logprobs, as the engine core sends
it: many requests, each with per-token tensors small enough that pyzmq
copies their frames, while the accumulated payload frame is large enough
that pyzmq sends it zero-copy."""
from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs
from vllm.v1.outputs import LogprobsTensors
outputs = []
for req in range(num_reqs):
num_tokens = num_prompt_tokens + req % 4
outputs.append(
EngineCoreOutput(
request_id=f"req-{req:08d}",
new_token_ids=[req],
new_prompt_logprobs_tensors=LogprobsTensors(
logprob_token_ids=torch.arange(
num_tokens * 2, dtype=torch.int64
).view(num_tokens, 2),
logprobs=torch.zeros(num_tokens, 2, dtype=torch.float32),
selected_token_ranks=torch.zeros(num_tokens, dtype=torch.int32),
),
)
)
return EngineCoreOutputs(outputs=outputs)
def test_payload_buffer_reuse_does_not_corrupt_in_flight_messages():
"""The engine core recycles the msgpack payload buffer across messages
(`MsgpackEncoder.encode_into`). It may only do so once zmq has finished
sending that buffer, otherwise a newer payload is delivered alongside the
older message's zero-copy tensor frames.
`Socket.send_multipart(track=True)` cannot be used to detect this: it
returns a tracker for the last frame only, and pyzmq copies frames below
`zmq.COPY_THRESHOLD` and reports them as already-sent.
"""
import zmq
from vllm.v1.engine import EngineCoreOutputs
from vllm.v1.engine.core import EngineCoreProc
num_msgs = 100
encoder = MsgpackEncoder()
decoder = MsgpackDecoder(EngineCoreOutputs)
# Enough requests that the payload frame is zero-copied rather than copied
# by pyzmq, which is what makes early reuse observable.
messages = [_logprobs_outputs(300, 24 + i % 8) for i in range(num_msgs)]
assert len(encoder.encode(messages[0])[0]) >= zmq.COPY_THRESHOLD
reuse_buffers: list[bytearray] = []
pending: list[tuple[zmq.MessageTracker, bytearray]] = []
with zmq.Context() as ctx:
push = ctx.socket(zmq.PUSH)
push.bind("inproc://test-payload-reuse")
pull = ctx.socket(zmq.PULL)
pull.connect("inproc://test-payload-reuse")
for outputs in messages:
while pending and pending[0][0].done:
reuse_buffers.append(pending.pop(0)[1])
buffer = reuse_buffers.pop() if reuse_buffers else bytearray()
buffers = encoder.encode_into(outputs, buffer)
tracker = EngineCoreProc._send_msg_tracking_payload(push, buffers)
if tracker.done:
reuse_buffers.append(buffer)
else:
pending.append((tracker, buffer))
for i, sent in enumerate(messages):
received = decoder.decode(pull.recv_multipart(copy=False))
assert len(received.outputs) == len(sent.outputs), f"message {i}"
for expected, actual in zip(sent.outputs, received.outputs):
sent_ids = expected.new_prompt_logprobs_tensors.logprob_token_ids
got_ids = actual.new_prompt_logprobs_tensors.logprob_token_ids
assert actual.request_id == expected.request_id, f"message {i}"
assert torch.equal(got_ids, sent_ids), (
f"message {i} request {actual.request_id}: corrupted "
f"prompt logprobs, {got_ids.shape} vs {sent_ids.shape}"
)
push.close(linger=0)
pull.close(linger=0)
def test_zero_copy_frames_survive_without_caller_side_references():
"""Callers don't need to retain the encoded object until zmq has sent it:
for a zero-copy frame, zmq holds its own reference to the backing buffer.
The engine core clients rely on this when sending requests that carry
tensors (e.g. prompt embeds) without tracking the messages.
What makes that safe is that `tensor_data()` hands zmq a memoryview which
transitively references the source tensor, so refcounting - not timing -
keeps the memory from being freed and reused underneath zmq.
"""
import gc
import zmq
from vllm.v1.utils import tensor_data
num_elems = 100_000 # comfortably over zmq.COPY_THRESHOLD
expected = torch.arange(num_elems, dtype=torch.int64)
encoder = MsgpackEncoder()
decoder = MsgpackDecoder(RequestWithTensor)
# The buffer handed to zmq must keep the tensor's storage alive by itself.
holder = tensor_data(expected).obj
while getattr(holder, "base", None) is not None:
holder = holder.base
assert isinstance(holder, torch.Tensor)
assert holder.data_ptr() == expected.data_ptr()
with zmq.Context() as ctx:
push = ctx.socket(zmq.PUSH)
push.bind("inproc://test-zero-copy-lifetime")
pull = ctx.socket(zmq.PULL)
pull.connect("inproc://test-zero-copy-lifetime")
request = RequestWithTensor(prompt_embeds=expected.clone(), data="req")
buffers = encoder.encode(request)
assert max(len(buf) for buf in buffers) >= zmq.COPY_THRESHOLD
push.send_multipart(buffers, copy=False)
# Drop every reference the sender holds, then churn the allocator.
del request, buffers
gc.collect()
torch.arange(num_elems * 4, dtype=torch.int64)
decoded = decoder.decode(pull.recv_multipart(copy=False))
assert decoded.prompt_embeds is not None
assert torch.equal(decoded.prompt_embeds, expected)
push.close(linger=0)
pull.close(linger=0)