diff --git a/vllm/distributed/weight_transfer/ipc_engine.py b/vllm/distributed/weight_transfer/ipc_engine.py index 8a7d8744c8..ed7fb86fa2 100644 --- a/vllm/distributed/weight_transfer/ipc_engine.py +++ b/vllm/distributed/weight_transfer/ipc_engine.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: from vllm.config import VllmConfig from vllm.distributed.weight_transfer.packed_tensor import ( DEFAULT_PACKED_BUFFER_SIZE_BYTES, + PackedBufferImporter, packed_ipc_consumer, packed_ipc_producer, ) @@ -143,6 +144,9 @@ class IPCWeightTransferEngine( # Set from the trainer-supplied init info at the handshake; defaults are # only for the (unreachable) receive-before-init case. self.packed = False + # Shared across all chunks of every packed transfer this engine + # receives; see PackedBufferImporter for the refcount contract. + self._packed_importer = PackedBufferImporter() def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None: """ @@ -170,6 +174,12 @@ class IPCWeightTransferEngine( ) finalize_layerwise_reload(self.model, self.model_config) + # Every reduce_tensor call is a fresh export with its own refcount + # slot, so releasing once per update always balances this update's + # export and lets the trainer reclaim its staging buffer. Callers + # that skip finish are still covered by the replace-on-next-export + # path inside the importer. + self._packed_importer.close() def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None: """ @@ -202,6 +212,7 @@ class IPCWeightTransferEngine( dtype_names=update_info.dtype_names, tensor_sizes=update_info.tensor_sizes, device_index=device_index, + importer=self._packed_importer, ) else: assert isinstance(update_info.ipc_handles, list) @@ -236,7 +247,7 @@ class IPCWeightTransferEngine( self.model.load_weights(weights) def shutdown(self) -> None: - pass + self._packed_importer.close() @staticmethod def trainer_send_weights(*args: Any, **kwargs: Any) -> None: diff --git a/vllm/distributed/weight_transfer/packed_tensor.py b/vllm/distributed/weight_transfer/packed_tensor.py index 59bdbafd40..79641a8395 100644 --- a/vllm/distributed/weight_transfer/packed_tensor.py +++ b/vllm/distributed/weight_transfer/packed_tensor.py @@ -381,6 +381,45 @@ def packed_ipc_producer( ) +class PackedBufferImporter: + """One-slot cache for the consumer-side mapping of a packed IPC buffer. + + torch's cross-process refcount pairs one ``reduce_tensor`` export with + one consumer rebuild-release cycle. The producer exports its buffer once + and ships the same args with every chunk, so rebuilding and releasing + per chunk decrements the counter once per chunk, drives it negative, + and the dropped buffer is then never reclaimable by ``ipc_collect`` on + the producer side (it leaks one buffer per transfer). Caching the + rebuilt buffer keyed by the rebuild args restores the pairing: chunks + of one transfer reuse the mapping (it aliases producer memory, so it + always reads the current chunk's bytes), and replacing the entry when + the next export arrives — or ``close()`` — releases the previous + mapping exactly once. + + The importer that consumes a multi-chunk transfer must be one object + reused across chunks; a fresh importer per chunk reintroduces the + per-chunk release. ``IPCWeightTransferEngine`` owns one instance per + engine and closes it on shutdown. + """ + + def __init__(self) -> None: + self._entry: tuple[tuple, torch.Tensor] | None = None + + def rebuild(self, list_args: list) -> torch.Tensor: + from torch.multiprocessing.reductions import rebuild_cuda_tensor + + key = tuple(list_args) + if self._entry is not None and self._entry[0] == key: + return self._entry[1] + packed = rebuild_cuda_tensor(*list_args) + self._entry = (key, packed) + return packed + + def close(self) -> None: + """Release the held mapping (one producer-side refcount decrement).""" + self._entry = None + + def packed_ipc_consumer( ipc_handle: dict[str, tuple], names: list[str], @@ -388,6 +427,7 @@ def packed_ipc_consumer( dtype_names: list[str], tensor_sizes: list[int], device_index: int, + importer: "PackedBufferImporter | None" = None, ) -> list[tuple[str, torch.Tensor]]: """Unpack a single packed IPC chunk into named tensors. @@ -410,9 +450,11 @@ def packed_ipc_consumer( dtype_names: Parameter dtype name strings (e.g. "float16") tensor_sizes: Size in bytes of each parameter in the packed buffer device_index: Local CUDA device index + importer: Import cache that must be shared across all chunks of one + export so the buffer mapping is opened and released exactly once + (see ``PackedBufferImporter``). ``None`` builds an ephemeral one, + which is only safe for single-chunk consumption. """ - from torch.multiprocessing.reductions import rebuild_cuda_tensor - props = torch.cuda.get_device_properties(device_index) physical_gpu_id = str(props.uuid) @@ -425,7 +467,9 @@ def packed_ipc_consumer( args = ipc_handle[physical_gpu_id] list_args = list(args) list_args[6] = device_index - packed = rebuild_cuda_tensor(*list_args) + if importer is None: + importer = PackedBufferImporter() + packed = importer.rebuild(list_args) content_size = sum(tensor_sizes) packed = packed[:content_size]