[Core] Ensure memory is pinned prior to async h2d copy (#45424)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-20 20:02:24 -07:00
committed by GitHub
parent 8dd1b702f2
commit 7df3d7dada
49 changed files with 254 additions and 264 deletions
+3 -2
View File
@@ -66,6 +66,7 @@ from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.vision import is_vit_use_data_parallel
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.moonvit import MoonViTConfig
from vllm.utils.torch_utils import async_tensor_h2d
def _apply_rope_input_validation(x, freqs_cis):
@@ -758,7 +759,7 @@ class MoonVitPretrainedModel(PreTrainedModel):
),
]
)
metadata["cu_seqlens"] = torch.from_numpy(cu_seqlens_np).to(device)
metadata["cu_seqlens"] = async_tensor_h2d(cu_seqlens_np, device=device)
if max_seqlen_override is not None:
max_seqlen_val = int(max_seqlen_override)
@@ -770,7 +771,7 @@ class MoonVitPretrainedModel(PreTrainedModel):
metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)
gather_idx_np = _build_merge_gather_idx(grid_pairs, self.merge_kernel_size)
metadata["merge_gather_idx"] = torch.from_numpy(gather_idx_np).to(device)
metadata["merge_gather_idx"] = async_tensor_h2d(gather_idx_np, device=device)
return metadata
+2 -3
View File
@@ -83,9 +83,8 @@ from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
@@ -825,7 +824,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
@staticmethod
def invert_permutation(perm: torch.Tensor) -> torch.Tensor:
# building the inverse permutation in O(n) time
inv = torch.empty_like(perm, pin_memory=is_pin_memory_available())
inv = torch.empty_like(perm, pin_memory=PIN_MEMORY)
inv[perm] = torch.arange(perm.numel(), device=perm.device, dtype=perm.dtype)
return inv