diff --git a/vllm/distributed/weight_transfer/packed_tensor.py b/vllm/distributed/weight_transfer/packed_tensor.py index 79641a8395..0eb9ede263 100644 --- a/vllm/distributed/weight_transfer/packed_tensor.py +++ b/vllm/distributed/weight_transfer/packed_tensor.py @@ -5,6 +5,7 @@ import math from collections.abc import Callable, Iterator from dataclasses import dataclass +from functools import cache from typing import Any import torch @@ -15,6 +16,14 @@ DEFAULT_PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024 # 1GB DEFAULT_PACKED_NUM_BUFFERS = 2 +# Streams are cached across calls: the caching allocator keys cached blocks by the +# stream they were allocated on, so fresh streams at every call would strand the +# packed buffers' reserved memory per call. See gh-52950. +@cache +def _get_streams(device_idx: int, num_buffers: int) -> tuple[torch.cuda.Stream, ...]: + return tuple(torch.cuda.Stream(device=device_idx) for _ in range(num_buffers)) + + def unpack_tensor( packed_tensor: torch.Tensor, names: list[str], @@ -154,7 +163,7 @@ def packed_nccl_broadcast_producer( Both producer and consumer must use the same value. """ - streams = [torch.cuda.Stream() for _ in range(num_buffers)] + streams = _get_streams(torch.accelerator.current_device_index(), num_buffers) # Keep references to in-flight chunks so their packed_tensors # aren't freed while an async broadcast is still reading them. in_flight: list[PackedChunk | None] = [None] * num_buffers @@ -209,7 +218,7 @@ def packed_nccl_broadcast_consumer( """ target_packed_tensor_size = buffer_size_bytes - streams = [torch.cuda.Stream() for _ in range(num_buffers)] + streams = _get_streams(torch.accelerator.current_device_index(), num_buffers) buffer_idx = 0 packing_tensor_meta_data: list[list[tuple[str, list[int], torch.dtype, int]]] = [