[Core] Ensure memory is pinned prior to async h2d copy (#45424)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-20 20:02:24 -07:00
committed by GitHub
parent 8dd1b702f2
commit 7df3d7dada
49 changed files with 254 additions and 264 deletions
+18 -15
View File
@@ -3,7 +3,6 @@
import contextlib
import importlib.metadata
import os
import platform
import random
import threading
from collections.abc import Callable, Collection
@@ -18,6 +17,7 @@ from torch.library import Library, infer_schema
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.utils.platform_utils import is_pin_memory_available
if TYPE_CHECKING:
from vllm.config import ModelConfig
@@ -68,9 +68,7 @@ MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP = {
T = TypeVar("T")
# Pin memory in non-WSL case.
# Logic duplicated here for now to avoid circular import.
PIN_MEMORY = "microsoft" not in " ".join(platform.uname()).lower()
PIN_MEMORY = is_pin_memory_available()
def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
@@ -606,14 +604,24 @@ def create_kv_caches_with_random(
def async_tensor_h2d(
data: list,
dtype: torch.dtype,
data: list | np.ndarray | torch.Tensor,
device: str | torch.device,
pin_memory: bool = PIN_MEMORY,
dtype: torch.dtype | None = None,
) -> torch.Tensor:
"""Asynchronously create a tensor and copy it from host to device."""
t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
return t.to(device=device, non_blocking=True)
"""Copy list/numpy array/tensor async from host to device."""
if isinstance(data, np.ndarray):
data = torch.from_numpy(data)
if isinstance(data, torch.Tensor):
t = data.pin_memory() if PIN_MEMORY else data
else:
t = torch.tensor(data, dtype=dtype, pin_memory=PIN_MEMORY, device="cpu")
assert t.is_cpu
return t.to(device=device, dtype=dtype, non_blocking=True)
def np_to_pinned_tensor(array: np.ndarray) -> torch.Tensor:
t = torch.from_numpy(array)
return t.pin_memory() if PIN_MEMORY else t
def make_ndarray_with_pad(
@@ -914,11 +922,6 @@ def _encode_layer_name(layer_name: str) -> str | LayerName:
return LayerName(layer_name) if _USE_LAYERNAME else layer_name
# Supports xccl with PyTorch versions >= 2.8.0.dev for XPU platform
def supports_xccl() -> bool:
return torch.distributed.is_xccl_available()
# Supports XPU Graph with PyTorch versions >= 2.11.0.dev for XPU platform
def supports_xpu_graph() -> bool:
return is_torch_equal_or_newer("2.11.0.dev")