[Bugfix] Import each packed IPC export once on the consumer side (#51259)

Signed-off-by: acmore <hezj07+github@gmail.com>
This commit is contained in:
acmore
2026-08-11 10:05:27 +08:00
committed by GitHub
parent 33584901ba
commit f37dff22fc
2 changed files with 59 additions and 4 deletions
+12 -1
View File
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
PackedBufferImporter,
packed_ipc_consumer,
packed_ipc_producer,
)
@@ -143,6 +144,9 @@ class IPCWeightTransferEngine(
# Set from the trainer-supplied init info at the handshake; defaults are
# only for the (unreachable) receive-before-init case.
self.packed = False
# Shared across all chunks of every packed transfer this engine
# receives; see PackedBufferImporter for the refcount contract.
self._packed_importer = PackedBufferImporter()
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
"""
@@ -170,6 +174,12 @@ class IPCWeightTransferEngine(
)
finalize_layerwise_reload(self.model, self.model_config)
# Every reduce_tensor call is a fresh export with its own refcount
# slot, so releasing once per update always balances this update's
# export and lets the trainer reclaim its staging buffer. Callers
# that skip finish are still covered by the replace-on-next-export
# path inside the importer.
self._packed_importer.close()
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
"""
@@ -202,6 +212,7 @@ class IPCWeightTransferEngine(
dtype_names=update_info.dtype_names,
tensor_sizes=update_info.tensor_sizes,
device_index=device_index,
importer=self._packed_importer,
)
else:
assert isinstance(update_info.ipc_handles, list)
@@ -236,7 +247,7 @@ class IPCWeightTransferEngine(
self.model.load_weights(weights)
def shutdown(self) -> None:
pass
self._packed_importer.close()
@staticmethod
def trainer_send_weights(*args: Any, **kwargs: Any) -> None:
@@ -381,6 +381,45 @@ def packed_ipc_producer(
)
class PackedBufferImporter:
"""One-slot cache for the consumer-side mapping of a packed IPC buffer.
torch's cross-process refcount pairs one ``reduce_tensor`` export with
one consumer rebuild-release cycle. The producer exports its buffer once
and ships the same args with every chunk, so rebuilding and releasing
per chunk decrements the counter once per chunk, drives it negative,
and the dropped buffer is then never reclaimable by ``ipc_collect`` on
the producer side (it leaks one buffer per transfer). Caching the
rebuilt buffer keyed by the rebuild args restores the pairing: chunks
of one transfer reuse the mapping (it aliases producer memory, so it
always reads the current chunk's bytes), and replacing the entry when
the next export arrives — or ``close()`` — releases the previous
mapping exactly once.
The importer that consumes a multi-chunk transfer must be one object
reused across chunks; a fresh importer per chunk reintroduces the
per-chunk release. ``IPCWeightTransferEngine`` owns one instance per
engine and closes it on shutdown.
"""
def __init__(self) -> None:
self._entry: tuple[tuple, torch.Tensor] | None = None
def rebuild(self, list_args: list) -> torch.Tensor:
from torch.multiprocessing.reductions import rebuild_cuda_tensor
key = tuple(list_args)
if self._entry is not None and self._entry[0] == key:
return self._entry[1]
packed = rebuild_cuda_tensor(*list_args)
self._entry = (key, packed)
return packed
def close(self) -> None:
"""Release the held mapping (one producer-side refcount decrement)."""
self._entry = None
def packed_ipc_consumer(
ipc_handle: dict[str, tuple],
names: list[str],
@@ -388,6 +427,7 @@ def packed_ipc_consumer(
dtype_names: list[str],
tensor_sizes: list[int],
device_index: int,
importer: "PackedBufferImporter | None" = None,
) -> list[tuple[str, torch.Tensor]]:
"""Unpack a single packed IPC chunk into named tensors.
@@ -410,9 +450,11 @@ def packed_ipc_consumer(
dtype_names: Parameter dtype name strings (e.g. "float16")
tensor_sizes: Size in bytes of each parameter in the packed buffer
device_index: Local CUDA device index
importer: Import cache that must be shared across all chunks of one
export so the buffer mapping is opened and released exactly once
(see ``PackedBufferImporter``). ``None`` builds an ephemeral one,
which is only safe for single-chunk consumption.
"""
from torch.multiprocessing.reductions import rebuild_cuda_tensor
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
@@ -425,7 +467,9 @@ def packed_ipc_consumer(
args = ipc_handle[physical_gpu_id]
list_args = list(args)
list_args[6] = device_index
packed = rebuild_cuda_tensor(*list_args)
if importer is None:
importer = PackedBufferImporter()
packed = importer.rebuild(list_args)
content_size = sum(tensor_sizes)
packed = packed[:content_size]