[Perf] Ensure async h2d copies are pinned in more places (#55202)
This commit is contained in:
@@ -21,6 +21,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -80,10 +81,7 @@ class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer):
|
||||
dim=0,
|
||||
)
|
||||
lengths_cpu = spatial_shapes.prod(dim=-1).to(torch.int32)
|
||||
lengths = lengths_cpu.to(
|
||||
device=pixel_values.device,
|
||||
non_blocking=True,
|
||||
)
|
||||
lengths = async_tensor_h2d(lengths_cpu, pixel_values.device)
|
||||
|
||||
cu_seqlens = torch.zeros(
|
||||
lengths.numel() + 1,
|
||||
|
||||
@@ -538,7 +538,7 @@ class Ernie4_5_VisionTransformer(nn.Module):
|
||||
else:
|
||||
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
|
||||
|
||||
cu_seqlens = cu_seqlens.to(device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, device)
|
||||
|
||||
return {
|
||||
"rotary_pos_emb": rotary_pos_emb,
|
||||
@@ -1746,8 +1746,8 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
# Eager fallback: run the full pipeline (ViT + resampler). The result
|
||||
# is scattered directly, so it must be the post-merge embeddings.
|
||||
pixel_values = mm_kwargs["pixel_values"].type(self.vision_model.dtype)
|
||||
grid_thw = mm_kwargs["image_grid_thw"].to(
|
||||
self.vision_model.device, non_blocking=True
|
||||
grid_thw = async_tensor_h2d(
|
||||
mm_kwargs["image_grid_thw"], self.vision_model.device
|
||||
)
|
||||
image_features = self.vision_model(pixel_values, grid_thw)
|
||||
return self.resampler_model(image_features, grid_thw)
|
||||
@@ -1768,7 +1768,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
output = outputs["default"]
|
||||
assert batch_mm_kwargs is not None
|
||||
grid_thw_cpu = batch_mm_kwargs["image_grid_thw"]
|
||||
grid_thw = grid_thw_cpu.to(output.device, non_blocking=True)
|
||||
grid_thw = async_tensor_h2d(grid_thw_cpu, output.device)
|
||||
# The valid token count slices the graph output for the eager
|
||||
# resampler call, so it has to come back to the host.
|
||||
num_valid = int(
|
||||
|
||||
@@ -70,6 +70,7 @@ from vllm.model_executor.models.glm4_1v import (
|
||||
Glm4vVisionTransformer,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
from .utils import (
|
||||
maybe_prefix,
|
||||
@@ -344,8 +345,11 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
|
||||
cu_seqlens = torch.repeat_interleave(
|
||||
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
|
||||
).cumsum(dim=0, dtype=torch.int32)
|
||||
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
|
||||
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
|
||||
# cat straight into a pinned buffer so the H2D copy stays non-blocking.
|
||||
cu_seqlens = torch.cat(
|
||||
[cu_seqlens.new_zeros(1), cu_seqlens],
|
||||
out=cu_seqlens.new_empty(cu_seqlens.numel() + 1, pin_memory=PIN_MEMORY),
|
||||
).to(self.device, non_blocking=True)
|
||||
|
||||
# pre-compute max_seqlen for attn mask to reduce cuMemcpy operations
|
||||
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
|
||||
|
||||
@@ -69,6 +69,7 @@ from vllm.transformers_utils.processors.isaac import (
|
||||
get_image_size_for_max_num_patches,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
|
||||
from .vision import is_vit_use_data_parallel
|
||||
|
||||
@@ -970,9 +971,14 @@ class IsaacForConditionalGeneration(
|
||||
device = next(self.language_model.parameters()).device
|
||||
dtype = self.vision_embedding.linear_fc1.weight.dtype
|
||||
pixel_values = pixel_values.to(device=device, dtype=dtype)
|
||||
spatial_grids = image_grid_thw[:, 1:3].to(
|
||||
device, dtype=torch.int32, non_blocking=True
|
||||
)
|
||||
# The [:, 1:3] column slice isn't densely laid out, so stage it with a
|
||||
# single copy straight into pinned memory to keep the H2D non-blocking.
|
||||
spatial_grids_cpu = torch.empty(
|
||||
(image_grid_thw.shape[0], 2),
|
||||
dtype=image_grid_thw.dtype,
|
||||
pin_memory=PIN_MEMORY,
|
||||
).copy_(image_grid_thw[:, 1:3])
|
||||
spatial_grids = async_tensor_h2d(spatial_grids_cpu, device, dtype=torch.int32)
|
||||
|
||||
vision_embeddings = self.vision_embedding((pixel_values, spatial_grids))
|
||||
merge_size = self.config.vision_config.pixel_shuffle_scale_factor
|
||||
|
||||
@@ -59,7 +59,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -1305,17 +1305,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
||||
)
|
||||
else:
|
||||
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
|
||||
# These are all built on the host, so stage them over
|
||||
# non-blocking rather than forcing a sync per transfer.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
# These are all built on the host; concat straight into pinned
|
||||
# buffers so the H2D copies stay non-blocking.
|
||||
siglip_position_ids = torch.concat(
|
||||
siglip_position_ids,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in siglip_position_ids),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values.device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values.device
|
||||
)
|
||||
sample_indices = torch.concat(sample_indices, dim=0).to(
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
sample_indices = torch.concat(
|
||||
sample_indices,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in sample_indices),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values.device, non_blocking=True)
|
||||
|
||||
image_embeds = self.visual(
|
||||
pixel_values=pixel_values,
|
||||
@@ -1359,16 +1371,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
||||
)
|
||||
else:
|
||||
pixel_values_videos = pixel_values_videos.type(self.visual.dtype)
|
||||
# These are host-built; stage them over non-blocking.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values_videos.device, non_blocking=True
|
||||
)
|
||||
# These are host-built; concat straight into pinned buffers so
|
||||
# the H2D copies stay non-blocking.
|
||||
siglip_position_ids = torch.concat(
|
||||
siglip_position_ids,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in siglip_position_ids),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values_videos.device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values_videos.device
|
||||
)
|
||||
sample_indices = torch.concat(sample_indices, dim=0).to(
|
||||
pixel_values_videos.device, non_blocking=True
|
||||
)
|
||||
sample_indices = torch.concat(
|
||||
sample_indices,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in sample_indices),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values_videos.device, non_blocking=True)
|
||||
|
||||
video_embeds = self.visual(
|
||||
pixel_values=pixel_values_videos,
|
||||
|
||||
@@ -68,7 +68,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .ernie45 import Ernie4_5ForCausalLM
|
||||
@@ -1142,10 +1142,17 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
|
||||
siglip_position_ids.append(image_position_ids)
|
||||
cu_seqlens.append(cu_seqlens[-1] + numel)
|
||||
|
||||
# Both are built on the host; stage them over non-blocking.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
# Both are built on the host; concat straight into a pinned buffer
|
||||
# so the H2D copy stays non-blocking.
|
||||
siglip_position_ids = torch.concat(
|
||||
siglip_position_ids,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in siglip_position_ids),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values.device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values.device
|
||||
)
|
||||
|
||||
@@ -643,8 +643,8 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
seq_len, batch_size, chunk_size_nc, left_chunk_nc
|
||||
)
|
||||
if device.type != "cpu":
|
||||
enc_streaming_mask_nc = enc_streaming_mask_nc.contiguous().to(
|
||||
device, non_blocking=True
|
||||
enc_streaming_mask_nc = async_tensor_h2d(
|
||||
enc_streaming_mask_nc.contiguous(), device
|
||||
)
|
||||
if masks is not None:
|
||||
hs_mask_nc = masks & enc_streaming_mask_nc
|
||||
|
||||
@@ -91,6 +91,7 @@ from vllm.multimodal.processing.processor import (
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -985,8 +986,8 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
input_features = audio_input["input_features"]
|
||||
# audio_feature_lengths is keep_on_cpu; the audio tower derives
|
||||
# device placement from feature_lens, so move it explicitly.
|
||||
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
|
||||
input_features.device, non_blocking=True
|
||||
audio_feature_lengths = async_tensor_h2d(
|
||||
audio_input["audio_feature_lengths"], input_features.device
|
||||
)
|
||||
|
||||
audio_feat_lengths, audio_output_lengths = (
|
||||
|
||||
@@ -1001,7 +1001,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
|
||||
|
||||
# Move cu_seqlens to GPU; grid_thw may be on CPU during profile_run
|
||||
# and FA3 vit attention requires cu_seqlens on CUDA.
|
||||
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, self.device)
|
||||
hidden_states = hidden_states.unsqueeze(1)
|
||||
rotary_pos_emb_cos = rotary_pos_emb_cos.to(hidden_states.device)
|
||||
rotary_pos_emb_sin = rotary_pos_emb_sin.to(hidden_states.device)
|
||||
@@ -1574,8 +1574,8 @@ class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMix
|
||||
input_features = audio_input["input_features"]
|
||||
# audio_feature_lengths is keep_on_cpu; the audio tower derives
|
||||
# device placement from feature_lens, so move it explicitly.
|
||||
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
|
||||
input_features.device, non_blocking=True
|
||||
audio_feature_lengths = async_tensor_h2d(
|
||||
audio_input["audio_feature_lengths"], input_features.device
|
||||
)
|
||||
|
||||
audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)
|
||||
|
||||
@@ -2367,8 +2367,8 @@ class Qwen3VLForConditionalGeneration(
|
||||
grid_thw_list = grid_thw.tolist()
|
||||
image_embeds_out = []
|
||||
for emb, size in zip(image_embeds_split, grid_thw_list):
|
||||
positions = compute_mrope_for_media(size, merge_size).to(
|
||||
emb.device, non_blocking=True
|
||||
positions = async_tensor_h2d(
|
||||
compute_mrope_for_media(size, merge_size), emb.device
|
||||
)
|
||||
positions = torch.cat(
|
||||
[
|
||||
@@ -2596,14 +2596,11 @@ class Qwen3VLForConditionalGeneration(
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=0, length=len(unpruned_token_ids)),
|
||||
)
|
||||
original_mrope = (
|
||||
self.get_mrope_input_positions(
|
||||
input_tokens=unpruned_token_ids,
|
||||
mm_features=[mm_feature],
|
||||
)[0]
|
||||
.to(device, non_blocking=True)
|
||||
.permute(1, 0)
|
||||
)
|
||||
original_mrope_cpu = self.get_mrope_input_positions(
|
||||
input_tokens=unpruned_token_ids,
|
||||
mm_features=[mm_feature],
|
||||
)[0]
|
||||
original_mrope = async_tensor_h2d(original_mrope_cpu, device).permute(1, 0)
|
||||
full_is_video_embed = unpruned_token_ids_tensor == embed_token_id
|
||||
|
||||
with gpu_sync_allowed():
|
||||
|
||||
Reference in New Issue
Block a user