From 26858770eccf9592073b502a57f4662aac358001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Mon, 24 Aug 2026 02:54:41 -0400 Subject: [PATCH] [Bugfix] Reuse CUDA streams in packed weight transfer to cap reserved-memory waste (#52951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Quentin Gallouédec --- vllm/distributed/weight_transfer/packed_tensor.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vllm/distributed/weight_transfer/packed_tensor.py b/vllm/distributed/weight_transfer/packed_tensor.py index 79641a8395..0eb9ede263 100644 --- a/vllm/distributed/weight_transfer/packed_tensor.py +++ b/vllm/distributed/weight_transfer/packed_tensor.py @@ -5,6 +5,7 @@ import math from collections.abc import Callable, Iterator from dataclasses import dataclass +from functools import cache from typing import Any import torch @@ -15,6 +16,14 @@ DEFAULT_PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024 # 1GB DEFAULT_PACKED_NUM_BUFFERS = 2 +# Streams are cached across calls: the caching allocator keys cached blocks by the +# stream they were allocated on, so fresh streams at every call would strand the +# packed buffers' reserved memory per call. See gh-52950. +@cache +def _get_streams(device_idx: int, num_buffers: int) -> tuple[torch.cuda.Stream, ...]: + return tuple(torch.cuda.Stream(device=device_idx) for _ in range(num_buffers)) + + def unpack_tensor( packed_tensor: torch.Tensor, names: list[str], @@ -154,7 +163,7 @@ def packed_nccl_broadcast_producer( Both producer and consumer must use the same value. """ - streams = [torch.cuda.Stream() for _ in range(num_buffers)] + streams = _get_streams(torch.accelerator.current_device_index(), num_buffers) # Keep references to in-flight chunks so their packed_tensors # aren't freed while an async broadcast is still reading them. in_flight: list[PackedChunk | None] = [None] * num_buffers @@ -209,7 +218,7 @@ def packed_nccl_broadcast_consumer( """ target_packed_tensor_size = buffer_size_bytes - streams = [torch.cuda.Stream() for _ in range(num_buffers)] + streams = _get_streams(torch.accelerator.current_device_index(), num_buffers) buffer_idx = 0 packing_tensor_meta_data: list[list[tuple[str, list[int], torch.dtype, int]]] = [