[Perf] Ensure async h2d copies are pinned in more places (#55202)

This commit is contained in:
Nick Hill
2026-09-04 17:17:01 -07:00
committed by GitHub
parent 874df9373d
commit 8277c42e4c
21 changed files with 206 additions and 73 deletions
+2 -4
View File
@@ -21,6 +21,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -80,10 +81,7 @@ class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer):
dim=0,
)
lengths_cpu = spatial_shapes.prod(dim=-1).to(torch.int32)
lengths = lengths_cpu.to(
device=pixel_values.device,
non_blocking=True,
)
lengths = async_tensor_h2d(lengths_cpu, pixel_values.device)
cu_seqlens = torch.zeros(
lengths.numel() + 1,
+4 -4
View File
@@ -538,7 +538,7 @@ class Ernie4_5_VisionTransformer(nn.Module):
else:
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
cu_seqlens = cu_seqlens.to(device, non_blocking=True)
cu_seqlens = async_tensor_h2d(cu_seqlens, device)
return {
"rotary_pos_emb": rotary_pos_emb,
@@ -1746,8 +1746,8 @@ class Ernie4_5_VLMoeForConditionalGeneration(
# Eager fallback: run the full pipeline (ViT + resampler). The result
# is scattered directly, so it must be the post-merge embeddings.
pixel_values = mm_kwargs["pixel_values"].type(self.vision_model.dtype)
grid_thw = mm_kwargs["image_grid_thw"].to(
self.vision_model.device, non_blocking=True
grid_thw = async_tensor_h2d(
mm_kwargs["image_grid_thw"], self.vision_model.device
)
image_features = self.vision_model(pixel_values, grid_thw)
return self.resampler_model(image_features, grid_thw)
@@ -1768,7 +1768,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
output = outputs["default"]
assert batch_mm_kwargs is not None
grid_thw_cpu = batch_mm_kwargs["image_grid_thw"]
grid_thw = grid_thw_cpu.to(output.device, non_blocking=True)
grid_thw = async_tensor_h2d(grid_thw_cpu, output.device)
# The valid token count slices the graph output for the eager
# resampler call, so it has to come back to the host.
num_valid = int(
+6 -2
View File
@@ -70,6 +70,7 @@ from vllm.model_executor.models.glm4_1v import (
Glm4vVisionTransformer,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.utils.torch_utils import PIN_MEMORY
from .utils import (
maybe_prefix,
@@ -344,8 +345,11 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
).cumsum(dim=0, dtype=torch.int32)
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
# cat straight into a pinned buffer so the H2D copy stays non-blocking.
cu_seqlens = torch.cat(
[cu_seqlens.new_zeros(1), cu_seqlens],
out=cu_seqlens.new_empty(cu_seqlens.numel() + 1, pin_memory=PIN_MEMORY),
).to(self.device, non_blocking=True)
# pre-compute max_seqlen for attn mask to reduce cuMemcpy operations
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
+9 -3
View File
@@ -69,6 +69,7 @@ from vllm.transformers_utils.processors.isaac import (
get_image_size_for_max_num_patches,
)
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from .vision import is_vit_use_data_parallel
@@ -970,9 +971,14 @@ class IsaacForConditionalGeneration(
device = next(self.language_model.parameters()).device
dtype = self.vision_embedding.linear_fc1.weight.dtype
pixel_values = pixel_values.to(device=device, dtype=dtype)
spatial_grids = image_grid_thw[:, 1:3].to(
device, dtype=torch.int32, non_blocking=True
)
# The [:, 1:3] column slice isn't densely laid out, so stage it with a
# single copy straight into pinned memory to keep the H2D non-blocking.
spatial_grids_cpu = torch.empty(
(image_grid_thw.shape[0], 2),
dtype=image_grid_thw.dtype,
pin_memory=PIN_MEMORY,
).copy_(image_grid_thw[:, 1:3])
spatial_grids = async_tensor_h2d(spatial_grids_cpu, device, dtype=torch.int32)
vision_embeddings = self.vision_embedding((pixel_values, spatial_grids))
merge_size = self.config.vision_config.pixel_shuffle_scale_factor
+41 -16
View File
@@ -59,7 +59,7 @@ from vllm.multimodal.processing import (
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -1305,17 +1305,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
)
else:
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
# These are all built on the host, so stage them over
# non-blocking rather than forcing a sync per transfer.
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
pixel_values.device, non_blocking=True
)
# These are all built on the host; concat straight into pinned
# buffers so the H2D copies stay non-blocking.
siglip_position_ids = torch.concat(
siglip_position_ids,
dim=0,
out=torch.empty(
sum(t.numel() for t in siglip_position_ids),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(
cu_seqlens, dtype=torch.int32, device=pixel_values.device
)
sample_indices = torch.concat(sample_indices, dim=0).to(
pixel_values.device, non_blocking=True
)
sample_indices = torch.concat(
sample_indices,
dim=0,
out=torch.empty(
sum(t.numel() for t in sample_indices),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values.device, non_blocking=True)
image_embeds = self.visual(
pixel_values=pixel_values,
@@ -1359,16 +1371,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
)
else:
pixel_values_videos = pixel_values_videos.type(self.visual.dtype)
# These are host-built; stage them over non-blocking.
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
pixel_values_videos.device, non_blocking=True
)
# These are host-built; concat straight into pinned buffers so
# the H2D copies stay non-blocking.
siglip_position_ids = torch.concat(
siglip_position_ids,
dim=0,
out=torch.empty(
sum(t.numel() for t in siglip_position_ids),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values_videos.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(
cu_seqlens, dtype=torch.int32, device=pixel_values_videos.device
)
sample_indices = torch.concat(sample_indices, dim=0).to(
pixel_values_videos.device, non_blocking=True
)
sample_indices = torch.concat(
sample_indices,
dim=0,
out=torch.empty(
sum(t.numel() for t in sample_indices),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values_videos.device, non_blocking=True)
video_embeds = self.visual(
pixel_values=pixel_values_videos,
+12 -5
View File
@@ -68,7 +68,7 @@ from vllm.multimodal.processing import (
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .ernie45 import Ernie4_5ForCausalLM
@@ -1142,10 +1142,17 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
siglip_position_ids.append(image_position_ids)
cu_seqlens.append(cu_seqlens[-1] + numel)
# Both are built on the host; stage them over non-blocking.
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
pixel_values.device, non_blocking=True
)
# Both are built on the host; concat straight into a pinned buffer
# so the H2D copy stays non-blocking.
siglip_position_ids = torch.concat(
siglip_position_ids,
dim=0,
out=torch.empty(
sum(t.numel() for t in siglip_position_ids),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(
cu_seqlens, dtype=torch.int32, device=pixel_values.device
)
+2 -2
View File
@@ -643,8 +643,8 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
seq_len, batch_size, chunk_size_nc, left_chunk_nc
)
if device.type != "cpu":
enc_streaming_mask_nc = enc_streaming_mask_nc.contiguous().to(
device, non_blocking=True
enc_streaming_mask_nc = async_tensor_h2d(
enc_streaming_mask_nc.contiguous(), device
)
if masks is not None:
hs_mask_nc = masks & enc_streaming_mask_nc
@@ -91,6 +91,7 @@ from vllm.multimodal.processing.processor import (
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -985,8 +986,8 @@ class Qwen2_5OmniConditionalGenerationMixin:
input_features = audio_input["input_features"]
# audio_feature_lengths is keep_on_cpu; the audio tower derives
# device placement from feature_lens, so move it explicitly.
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
input_features.device, non_blocking=True
audio_feature_lengths = async_tensor_h2d(
audio_input["audio_feature_lengths"], input_features.device
)
audio_feat_lengths, audio_output_lengths = (
@@ -1001,7 +1001,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
# Move cu_seqlens to GPU; grid_thw may be on CPU during profile_run
# and FA3 vit attention requires cu_seqlens on CUDA.
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(cu_seqlens, self.device)
hidden_states = hidden_states.unsqueeze(1)
rotary_pos_emb_cos = rotary_pos_emb_cos.to(hidden_states.device)
rotary_pos_emb_sin = rotary_pos_emb_sin.to(hidden_states.device)
@@ -1574,8 +1574,8 @@ class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMix
input_features = audio_input["input_features"]
# audio_feature_lengths is keep_on_cpu; the audio tower derives
# device placement from feature_lens, so move it explicitly.
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
input_features.device, non_blocking=True
audio_feature_lengths = async_tensor_h2d(
audio_input["audio_feature_lengths"], input_features.device
)
audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)
+7 -10
View File
@@ -2367,8 +2367,8 @@ class Qwen3VLForConditionalGeneration(
grid_thw_list = grid_thw.tolist()
image_embeds_out = []
for emb, size in zip(image_embeds_split, grid_thw_list):
positions = compute_mrope_for_media(size, merge_size).to(
emb.device, non_blocking=True
positions = async_tensor_h2d(
compute_mrope_for_media(size, merge_size), emb.device
)
positions = torch.cat(
[
@@ -2596,14 +2596,11 @@ class Qwen3VLForConditionalGeneration(
identifier="DUMMY",
mm_position=PlaceholderRange(offset=0, length=len(unpruned_token_ids)),
)
original_mrope = (
self.get_mrope_input_positions(
input_tokens=unpruned_token_ids,
mm_features=[mm_feature],
)[0]
.to(device, non_blocking=True)
.permute(1, 0)
)
original_mrope_cpu = self.get_mrope_input_positions(
input_tokens=unpruned_token_ids,
mm_features=[mm_feature],
)[0]
original_mrope = async_tensor_h2d(original_mrope_cpu, device).permute(1, 0)
full_is_video_embed = unpruned_token_ids_tensor == embed_token_id
with gpu_sync_allowed():