[Perf] Pin CPU tensors before non_blocking H2D in three MM paths (#54292)
Signed-off-by: Keyi Li <likey6688@gmail.com> Co-authored-by: Keyi Li <likey6688@gmail.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Keyi Li
Claude Opus 4.7
parent
93ab92be0c
commit
46a83642f6
@@ -259,10 +259,7 @@ class MMEncoderAttention(CustomOp):
|
||||
sequence_lengths = add_padding_to_seqlens(
|
||||
sequence_lengths, len(sequence_lengths), 0
|
||||
)
|
||||
sequence_lengths = torch.from_numpy(sequence_lengths).to(
|
||||
device, non_blocking=True
|
||||
)
|
||||
return sequence_lengths
|
||||
return async_tensor_h2d(sequence_lengths, device=device)
|
||||
|
||||
@classmethod
|
||||
def maybe_recompute_cu_seqlens(
|
||||
|
||||
@@ -38,6 +38,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
from .qwen2_vl import Qwen2VisionTransformer
|
||||
@@ -323,10 +324,7 @@ class CustomQwen2VLVE(Qwen2VisionTransformer):
|
||||
grid_thw_np[:, 0],
|
||||
).cumsum(axis=0, dtype=np.int32)
|
||||
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
|
||||
cu_seqlens = torch.from_numpy(cu_seqlens).to(
|
||||
self.device,
|
||||
non_blocking=True,
|
||||
)
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, device=self.device)
|
||||
|
||||
# Shape to (S, B, D) with batch dimension 1 as expected by the blocks.
|
||||
x = x.unsqueeze(1)
|
||||
|
||||
@@ -37,6 +37,7 @@ from vllm.model_executor.models.vision import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -821,9 +822,7 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
merge_gather_idx = build_image_merge_gather_idx(
|
||||
grid_thw_list, self.merge_kernel_size
|
||||
)
|
||||
metadata["merge_gather_idx"] = torch.from_numpy(merge_gather_idx).to(
|
||||
device=device, non_blocking=True
|
||||
)
|
||||
metadata["merge_gather_idx"] = async_tensor_h2d(merge_gather_idx, device=device)
|
||||
return metadata
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user