[Bugfix] Don't reuse engine core payload buffer while zmq is sending it (#50053)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
4fb483ca86
commit
6453fc0b8c
@@ -423,3 +423,139 @@ def test_multiple_senders_single_receiver_ipc():
|
||||
assert torch.allclose(decoded.prompt_embeds, original_tensor), (
|
||||
f"Value mismatch for sender {sender_idx} msg {msg_idx}"
|
||||
)
|
||||
|
||||
|
||||
def _logprobs_outputs(num_reqs: int, num_prompt_tokens: int):
|
||||
"""An EngineCoreOutputs carrying prompt logprobs, as the engine core sends
|
||||
it: many requests, each with per-token tensors small enough that pyzmq
|
||||
copies their frames, while the accumulated payload frame is large enough
|
||||
that pyzmq sends it zero-copy."""
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
|
||||
outputs = []
|
||||
for req in range(num_reqs):
|
||||
num_tokens = num_prompt_tokens + req % 4
|
||||
outputs.append(
|
||||
EngineCoreOutput(
|
||||
request_id=f"req-{req:08d}",
|
||||
new_token_ids=[req],
|
||||
new_prompt_logprobs_tensors=LogprobsTensors(
|
||||
logprob_token_ids=torch.arange(
|
||||
num_tokens * 2, dtype=torch.int64
|
||||
).view(num_tokens, 2),
|
||||
logprobs=torch.zeros(num_tokens, 2, dtype=torch.float32),
|
||||
selected_token_ranks=torch.zeros(num_tokens, dtype=torch.int32),
|
||||
),
|
||||
)
|
||||
)
|
||||
return EngineCoreOutputs(outputs=outputs)
|
||||
|
||||
|
||||
def test_payload_buffer_reuse_does_not_corrupt_in_flight_messages():
|
||||
"""The engine core recycles the msgpack payload buffer across messages
|
||||
(`MsgpackEncoder.encode_into`). It may only do so once zmq has finished
|
||||
sending that buffer, otherwise a newer payload is delivered alongside the
|
||||
older message's zero-copy tensor frames.
|
||||
|
||||
`Socket.send_multipart(track=True)` cannot be used to detect this: it
|
||||
returns a tracker for the last frame only, and pyzmq copies frames below
|
||||
`zmq.COPY_THRESHOLD` and reports them as already-sent.
|
||||
"""
|
||||
import zmq
|
||||
|
||||
from vllm.v1.engine import EngineCoreOutputs
|
||||
from vllm.v1.engine.core import EngineCoreProc
|
||||
|
||||
num_msgs = 100
|
||||
encoder = MsgpackEncoder()
|
||||
decoder = MsgpackDecoder(EngineCoreOutputs)
|
||||
# Enough requests that the payload frame is zero-copied rather than copied
|
||||
# by pyzmq, which is what makes early reuse observable.
|
||||
messages = [_logprobs_outputs(300, 24 + i % 8) for i in range(num_msgs)]
|
||||
assert len(encoder.encode(messages[0])[0]) >= zmq.COPY_THRESHOLD
|
||||
|
||||
reuse_buffers: list[bytearray] = []
|
||||
pending: list[tuple[zmq.MessageTracker, bytearray]] = []
|
||||
with zmq.Context() as ctx:
|
||||
push = ctx.socket(zmq.PUSH)
|
||||
push.bind("inproc://test-payload-reuse")
|
||||
pull = ctx.socket(zmq.PULL)
|
||||
pull.connect("inproc://test-payload-reuse")
|
||||
|
||||
for outputs in messages:
|
||||
while pending and pending[0][0].done:
|
||||
reuse_buffers.append(pending.pop(0)[1])
|
||||
buffer = reuse_buffers.pop() if reuse_buffers else bytearray()
|
||||
buffers = encoder.encode_into(outputs, buffer)
|
||||
tracker = EngineCoreProc._send_msg_tracking_payload(push, buffers)
|
||||
if tracker.done:
|
||||
reuse_buffers.append(buffer)
|
||||
else:
|
||||
pending.append((tracker, buffer))
|
||||
|
||||
for i, sent in enumerate(messages):
|
||||
received = decoder.decode(pull.recv_multipart(copy=False))
|
||||
assert len(received.outputs) == len(sent.outputs), f"message {i}"
|
||||
for expected, actual in zip(sent.outputs, received.outputs):
|
||||
sent_ids = expected.new_prompt_logprobs_tensors.logprob_token_ids
|
||||
got_ids = actual.new_prompt_logprobs_tensors.logprob_token_ids
|
||||
assert actual.request_id == expected.request_id, f"message {i}"
|
||||
assert torch.equal(got_ids, sent_ids), (
|
||||
f"message {i} request {actual.request_id}: corrupted "
|
||||
f"prompt logprobs, {got_ids.shape} vs {sent_ids.shape}"
|
||||
)
|
||||
push.close(linger=0)
|
||||
pull.close(linger=0)
|
||||
|
||||
|
||||
def test_zero_copy_frames_survive_without_caller_side_references():
|
||||
"""Callers don't need to retain the encoded object until zmq has sent it:
|
||||
for a zero-copy frame, zmq holds its own reference to the backing buffer.
|
||||
|
||||
The engine core clients rely on this when sending requests that carry
|
||||
tensors (e.g. prompt embeds) without tracking the messages.
|
||||
|
||||
What makes that safe is that `tensor_data()` hands zmq a memoryview which
|
||||
transitively references the source tensor, so refcounting - not timing -
|
||||
keeps the memory from being freed and reused underneath zmq.
|
||||
"""
|
||||
import gc
|
||||
|
||||
import zmq
|
||||
|
||||
from vllm.v1.utils import tensor_data
|
||||
|
||||
num_elems = 100_000 # comfortably over zmq.COPY_THRESHOLD
|
||||
expected = torch.arange(num_elems, dtype=torch.int64)
|
||||
encoder = MsgpackEncoder()
|
||||
decoder = MsgpackDecoder(RequestWithTensor)
|
||||
|
||||
# The buffer handed to zmq must keep the tensor's storage alive by itself.
|
||||
holder = tensor_data(expected).obj
|
||||
while getattr(holder, "base", None) is not None:
|
||||
holder = holder.base
|
||||
assert isinstance(holder, torch.Tensor)
|
||||
assert holder.data_ptr() == expected.data_ptr()
|
||||
|
||||
with zmq.Context() as ctx:
|
||||
push = ctx.socket(zmq.PUSH)
|
||||
push.bind("inproc://test-zero-copy-lifetime")
|
||||
pull = ctx.socket(zmq.PULL)
|
||||
pull.connect("inproc://test-zero-copy-lifetime")
|
||||
|
||||
request = RequestWithTensor(prompt_embeds=expected.clone(), data="req")
|
||||
buffers = encoder.encode(request)
|
||||
assert max(len(buf) for buf in buffers) >= zmq.COPY_THRESHOLD
|
||||
push.send_multipart(buffers, copy=False)
|
||||
|
||||
# Drop every reference the sender holds, then churn the allocator.
|
||||
del request, buffers
|
||||
gc.collect()
|
||||
torch.arange(num_elems * 4, dtype=torch.int64)
|
||||
|
||||
decoded = decoder.decode(pull.recv_multipart(copy=False))
|
||||
assert decoded.prompt_embeds is not None
|
||||
assert torch.equal(decoded.prompt_embeds, expected)
|
||||
push.close(linger=0)
|
||||
pull.close(linger=0)
|
||||
|
||||
Reference in New Issue
Block a user