[Perf] Pin CPU tensors before non_blocking H2D in three MM paths (#54292)

Signed-off-by: Keyi Li <likey6688@gmail.com>
Co-authored-by: Keyi Li <likey6688@gmail.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Keyi Li
2026-08-28 20:13:40 -07:00
committed by GitHub
co-authored by Keyi Li Claude Opus 4.7
parent 93ab92be0c
commit 46a83642f6
3 changed files with 5 additions and 11 deletions
@@ -259,10 +259,7 @@ class MMEncoderAttention(CustomOp):
sequence_lengths = add_padding_to_seqlens(
sequence_lengths, len(sequence_lengths), 0
)
sequence_lengths = torch.from_numpy(sequence_lengths).to(
device, non_blocking=True
)
return sequence_lengths
return async_tensor_h2d(sequence_lengths, device=device)
@classmethod
def maybe_recompute_cu_seqlens(
+2 -4
View File
@@ -38,6 +38,7 @@ from vllm.multimodal.processing import (
from vllm.sequence import IntermediateTensors
from vllm.utils.import_utils import resolve_obj_by_qualname
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from .qwen2_vl import Qwen2VisionTransformer
@@ -323,10 +324,7 @@ class CustomQwen2VLVE(Qwen2VisionTransformer):
grid_thw_np[:, 0],
).cumsum(axis=0, dtype=np.int32)
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
cu_seqlens = torch.from_numpy(cu_seqlens).to(
self.device,
non_blocking=True,
)
cu_seqlens = async_tensor_h2d(cu_seqlens, device=self.device)
# Shape to (S, B, D) with batch dimension 1 as expected by the blocks.
x = x.unsqueeze(1)
+2 -3
View File
@@ -37,6 +37,7 @@ from vllm.model_executor.models.vision import (
)
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
from vllm.utils.torch_utils import async_tensor_h2d
logger = init_logger(__name__)
@@ -821,9 +822,7 @@ class MoonViT3dPretrainedModel(nn.Module):
merge_gather_idx = build_image_merge_gather_idx(
grid_thw_list, self.merge_kernel_size
)
metadata["merge_gather_idx"] = torch.from_numpy(merge_gather_idx).to(
device=device, non_blocking=True
)
metadata["merge_gather_idx"] = async_tensor_h2d(merge_gather_idx, device=device)
return metadata