[Perf][3/n] Eliminate GPU<->CPU syncs in attention impls (#41434)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-07 19:44:24 -07:00
committed by GitHub
parent cd58e30872
commit 989c176c0a
10 changed files with 169 additions and 68 deletions
+9 -3
View File
@@ -3,6 +3,7 @@
import contextlib
import importlib.metadata
import os
import platform
import random
import threading
from collections.abc import Callable, Collection
@@ -67,6 +68,11 @@ MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP = {
T = TypeVar("T")
# Pin memory in non-WSL case.
# Logic duplicated here for now to avoid circular import.
PIN_MEMORY = "microsoft" not in " ".join(platform.uname()).lower()
def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
return (
kv_cache_dtype.startswith("fp8")
@@ -602,12 +608,12 @@ def create_kv_caches_with_random(
def async_tensor_h2d(
data: list,
dtype: torch.dtype,
target_device: str | torch.device,
pin_memory: bool,
device: str | torch.device,
pin_memory: bool = PIN_MEMORY,
) -> torch.Tensor:
"""Asynchronously create a tensor and copy it from host to device."""
t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
return t.to(device=target_device, non_blocking=True)
return t.to(device=device, non_blocking=True)
def make_ndarray_with_pad(