[Bugfix] Reuse CUDA streams in packed weight transfer to cap reserved-memory waste (#52951)

Signed-off-by: Quentin Gallouédec <quentin.gallouedec@huggingface.co>
This commit is contained in:
Quentin Gallouédec
2026-08-23 23:54:41 -07:00
committed by GitHub
parent cc40c3673b
commit 26858770ec
@@ -5,6 +5,7 @@
import math
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from functools import cache
from typing import Any
import torch
@@ -15,6 +16,14 @@ DEFAULT_PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024 # 1GB
DEFAULT_PACKED_NUM_BUFFERS = 2
# Streams are cached across calls: the caching allocator keys cached blocks by the
# stream they were allocated on, so fresh streams at every call would strand the
# packed buffers' reserved memory per call. See gh-52950.
@cache
def _get_streams(device_idx: int, num_buffers: int) -> tuple[torch.cuda.Stream, ...]:
return tuple(torch.cuda.Stream(device=device_idx) for _ in range(num_buffers))
def unpack_tensor(
packed_tensor: torch.Tensor,
names: list[str],
@@ -154,7 +163,7 @@ def packed_nccl_broadcast_producer(
Both producer and consumer must use the same value.
"""
streams = [torch.cuda.Stream() for _ in range(num_buffers)]
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
# Keep references to in-flight chunks so their packed_tensors
# aren't freed while an async broadcast is still reading them.
in_flight: list[PackedChunk | None] = [None] * num_buffers
@@ -209,7 +218,7 @@ def packed_nccl_broadcast_consumer(
"""
target_packed_tensor_size = buffer_size_bytes
streams = [torch.cuda.Stream() for _ in range(num_buffers)]
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
buffer_idx = 0
packing_tensor_meta_data: list[list[tuple[str, list[int], torch.dtype, int]]] = [