[Perf][4/n] Eliminate various GPU<->CPU syncs (#42347)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-19 10:35:54 -04:00
committed by GitHub
parent a78b842d0e
commit b82e908b4c
23 changed files with 129 additions and 108 deletions
+3 -6
View File
@@ -559,13 +559,10 @@ def _encode_token_type_ids(
def _decode_token_type_ids(input_ids: torch.Tensor) -> torch.Tensor:
ids_mask = (
torch.ones_like(input_ids, dtype=torch.int32, device=input_ids.device)
<< TOKEN_TYPE_SHIFT
)
tokens_mask = ids_mask.bitwise_not()
ids_mask = 1 << TOKEN_TYPE_SHIFT
tokens_mask = ~ids_mask
token_type_ids = input_ids.bitwise_and(ids_mask) >> TOKEN_TYPE_SHIFT
token_type_ids = (input_ids & ids_mask) >> TOKEN_TYPE_SHIFT
input_ids.bitwise_and_(tokens_mask)
+1 -1
View File
@@ -305,7 +305,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
return dict(
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),
num_patches=MultiModalFieldConfig.batched("image"),
num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
)
def _get_prompt_updates(
+1
View File
@@ -723,6 +723,7 @@ class Glm4vVisionTransformer(nn.Module):
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
pos_ids = pos_ids.to(cos.device, non_blocking=True)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
return cos_combined, sin_combined, pos_ids
+7 -7
View File
@@ -143,7 +143,7 @@ class GraniteSpeechMultiModalProcessor(
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
input_features=MultiModalFieldConfig.batched("audio"),
audio_embed_sizes=MultiModalFieldConfig.batched("audio"),
audio_embed_sizes=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
)
def _get_prompt_updates(
@@ -717,13 +717,13 @@ class GraniteSpeechForConditionalGeneration(
torch.Tensor: Mask of shape (bsz, num_features) to be applied to
the audio features prior to splitting the audio embeddings.
"""
most_audio_features = torch.max(audio_embed_sizes).item()
mask_indices = torch.arange(
most_audio_features,
device=audio_embed_sizes.device,
).view(1, -1)
most_audio_features = int(torch.max(audio_embed_sizes))
mask_indices = torch.arange(most_audio_features).view(1, -1)
input_features_mask = mask_indices < audio_embed_sizes.view(-1, 1)
return input_features_mask
target_device = self.encoder.input_linear.weight.device
if target_device == input_features_mask.device:
return input_features_mask
return input_features_mask.pin_memory().to(target_device, non_blocking=True)
def _pad_and_stack_input_features(
self,
+1 -1
View File
@@ -355,7 +355,7 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
"image", num_patches
),
image_embeds=MultiModalFieldConfig.batched("image"),
num_patches=MultiModalFieldConfig.batched("image"),
num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
)
def _get_prompt_updates(
+5 -3
View File
@@ -247,9 +247,11 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
"image", image_num_patches
),
image_num_patches=MultiModalFieldConfig.batched("image"),
image_num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
image_embeds=MultiModalFieldConfig.batched("image"),
image_token_id=MultiModalFieldConfig.shared("image", num_images),
image_token_id=MultiModalFieldConfig.shared(
"image", num_images, keep_on_cpu=True
),
)
def _get_mm_fields_config(
@@ -475,7 +477,7 @@ class InternVLMultiModalProcessor(
pixel_values_flat_video=MultiModalFieldConfig.flat_from_sizes(
"video", video_num_patches
),
video_num_patches=MultiModalFieldConfig.batched("video"),
video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
)
+1 -1
View File
@@ -214,7 +214,7 @@ class LlavaNextMultiModalProcessor(
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
pixel_values=MultiModalFieldConfig.batched("image"),
image_sizes=MultiModalFieldConfig.batched("image"),
image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
image_embeds=MultiModalFieldConfig.batched("image"),
)
@@ -316,7 +316,7 @@ class LlavaOnevisionMultiModalProcessor(
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
pixel_values=MultiModalFieldConfig.batched("image"),
image_sizes=MultiModalFieldConfig.batched("image"),
image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
image_embeds=MultiModalFieldConfig.batched("image"),
pixel_values_videos=MultiModalFieldConfig.batched("video"),
)
+9 -3
View File
@@ -586,7 +586,9 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
seq_len, batch_size, self.chunk_size, self.left_chunk
)
device = xs_pad.device
enc_streaming_mask = enc_streaming_mask.to(device)
enc_streaming_mask = enc_streaming_mask.contiguous().to(
device, non_blocking=True
)
xs_pad = xs_pad.to(device)
input_tensor = xs_pad
@@ -605,7 +607,9 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
seq_len, batch_size, chunk_size_nc, left_chunk_nc
)
if device.type != "cpu":
enc_streaming_mask_nc = enc_streaming_mask_nc.to(device)
enc_streaming_mask_nc = enc_streaming_mask_nc.contiguous().to(
device, non_blocking=True
)
if masks is not None:
hs_mask_nc = masks & enc_streaming_mask_nc
else:
@@ -917,7 +921,9 @@ class ConformerEncoder(TransformerEncoderBase):
enc_streaming_mask = self._streaming_mask(
max_audio_length, batch_size, self.chunk_size, self.left_chunk
)
enc_streaming_mask = enc_streaming_mask.to(device)
enc_streaming_mask = enc_streaming_mask.contiguous().to(
device, non_blocking=True
)
if mask is None:
return enc_streaming_mask
@@ -278,15 +278,15 @@ def create_qwen2_5_omni_thinker_field_factory(
image_embeds=MultiModalFieldConfig.flat_from_sizes(
"image", image_embed_grid_sizes
),
image_grid_thw=MultiModalFieldConfig.batched("image"),
image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
"video", video_grid_sizes
),
video_embeds=MultiModalFieldConfig.flat_from_sizes(
"video", video_embed_grid_sizes
),
video_grid_thw=MultiModalFieldConfig.batched("video"),
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
use_audio_in_video=MultiModalFieldConfig.shared("video", num_videos),
)
+12 -7
View File
@@ -84,6 +84,7 @@ from vllm.multimodal.processing import PromptReplacement, PromptUpdate
from vllm.sequence import IntermediateTensors
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
@@ -679,6 +680,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_size)
pos_ids = pos_ids.to(cos.device, non_blocking=True)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
@@ -737,9 +739,10 @@ class Qwen2_5_VisionTransformer(nn.Module):
window_index_thw, cu_seqlens_window_thw = self.get_window_index_thw(t, h, w)
cos_thw, sin_thw = self.rotary_pos_emb_thw(t, h, w)
cos_thw = cos_thw[window_index_thw, :, :]
window_index_thw_dev = window_index_thw.to(cos_thw.device, non_blocking=True)
cos_thw = cos_thw[window_index_thw_dev, :, :]
cos_thw = cos_thw.flatten(start_dim=0, end_dim=1)
sin_thw = sin_thw[window_index_thw, :, :]
sin_thw = sin_thw[window_index_thw_dev, :, :]
sin_thw = sin_thw.flatten(start_dim=0, end_dim=1)
cu_seqlens_thw = torch.repeat_interleave(
@@ -1036,7 +1039,7 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
**super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
)
def _call_hf_processor(
@@ -1361,7 +1364,9 @@ class Qwen2_5_VLForConditionalGeneration(
grid_thw_list = grid_thw.tolist()
image_embeds_out = []
for emb, size in zip(image_embeds_split, grid_thw_list):
positions = compute_mrope_for_media(size, merge_size).to(emb.device)
positions = compute_mrope_for_media(size, merge_size).to(
emb.device, non_blocking=True
)
emb = torch.cat([emb, positions], dim=1)
image_embeds_out.append(emb)
image_embeds_split = image_embeds_out
@@ -1443,7 +1448,7 @@ class Qwen2_5_VLForConditionalGeneration(
merge_size,
tokens_per_second=tokens_per_second,
video_second_per_grid=video_second_per_grid_t.item(),
).to(emb.device)
).to(emb.device, non_blocking=True)
emb = emb[retention_mask]
positions = positions[retention_mask]
@@ -1488,8 +1493,8 @@ class Qwen2_5_VLForConditionalGeneration(
else mrope_positions.device
)
# Tensors
input_ids_t = torch.as_tensor(input_ids, device=device, dtype=torch.long)
# Tensors.
input_ids_t = async_tensor_h2d(input_ids, dtype=torch.long, device=device)
mm_embeddings_out = [mm[:, :-4] for mm in multimodal_embeddings]
mm_embeddings_pos = [
+3 -3
View File
@@ -439,9 +439,9 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal, Supports
num_audios, max_audio_tokens, embed_dim = audio_features.shape
audio_output_lengths = audio_output_lengths.unsqueeze(1)
audio_features_mask = (
torch.arange(max_audio_tokens)
.expand(num_audios, max_audio_tokens)
.to(audio_output_lengths.device)
torch.arange(max_audio_tokens, device=audio_output_lengths.device).expand(
num_audios, max_audio_tokens
)
< audio_output_lengths
)
masked_audio_features = audio_features[audio_features_mask].view(-1, embed_dim)
+1
View File
@@ -634,6 +634,7 @@ class Qwen2VisionTransformer(nn.Module):
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
pos_ids = pos_ids.to(cos.device, non_blocking=True)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
return cos_combined, sin_combined
@@ -80,6 +80,7 @@ from vllm.multimodal.processing.processor import (
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interfaces import (
@@ -502,9 +503,9 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
cu_chunk_lens.extend([window_aftercnn] * num_full_chunks)
if remainder:
cu_chunk_lens.append(remainder)
cu_seqlens = torch.tensor(cu_chunk_lens, device=aftercnn_lens.device).cumsum(
-1, dtype=torch.int32
)
cu_seqlens = async_tensor_h2d(
cu_chunk_lens, dtype=torch.int32, device=aftercnn_lens.device
).cumsum(-1, dtype=torch.int32)
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
@@ -861,6 +862,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
pos_ids = pos_ids.to(cos.device, non_blocking=True)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
+6 -3
View File
@@ -101,6 +101,7 @@ from vllm.utils.collection_utils import is_list_of
from vllm.utils.math_utils import round_up
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
from ...utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
SupportsEagle,
@@ -2157,7 +2158,9 @@ class Qwen3VLForConditionalGeneration(
grid_thw_list = grid_thw.tolist()
image_embeds_out = []
for emb, size in zip(image_embeds_split, grid_thw_list):
positions = compute_mrope_for_media(size, merge_size).to(emb.device)
positions = compute_mrope_for_media(size, merge_size).to(
emb.device, non_blocking=True
)
positions = torch.cat(
[
positions,
@@ -2388,7 +2391,7 @@ class Qwen3VLForConditionalGeneration(
input_tokens=unpruned_token_ids,
mm_features=[mm_feature],
)[0]
.to(device)
.to(device, non_blocking=True)
.permute(1, 0)
)
full_is_video_embed = unpruned_token_ids_tensor == embed_token_id
@@ -2636,7 +2639,7 @@ class Qwen3VLForConditionalGeneration(
)
# Tensors
input_ids_t = torch.as_tensor(input_ids, device=device, dtype=torch.long)
input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long)
mm_embeddings_out = []
mm_embeddings_pos = []
@@ -152,7 +152,9 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
# Keep these as batched, as they always have batch size as first dim
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched("image")
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
return mm_fields
def _get_hf_mm_data(
+7 -15
View File
@@ -30,10 +30,8 @@ from vllm.model_executor.models.interfaces import supports_any_eagle
from vllm.multimodal import NestedTensors
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import (
is_pin_memory_available,
)
from vllm.utils.torch_utils import (
async_tensor_h2d,
direct_register_custom_op,
)
@@ -498,10 +496,9 @@ def isin_list(
elements: torch.Tensor,
test_elements_list: list[int],
) -> torch.Tensor:
test_elements = torch.tensor(
test_elements_list,
pin_memory=is_pin_memory_available(),
).to(device=elements.device, non_blocking=True)
test_elements = async_tensor_h2d(
test_elements_list, dtype=torch.int64, device=elements.device
)
return torch.isin(elements, test_elements)
@@ -772,14 +769,9 @@ def extract_layer_index(layer_name: str, num_attn_module: int = 1) -> int:
return layer_index
def cast_overflow_tensors(
tensors: torch.Tensor,
offset: float = 1000,
) -> torch.Tensor:
if tensors.isinf().any() or tensors.isnan().any():
clamp_value = torch.finfo(tensors.dtype).max - offset
tensors = torch.clamp(tensors, min=-clamp_value, max=clamp_value)
return tensors
def cast_overflow_tensors(tensors: torch.Tensor, offset: float = 1000) -> torch.Tensor:
clamp_value = torch.finfo(tensors.dtype).max - offset
return torch.clamp(tensors, min=-clamp_value, max=clamp_value)
def fast_topk(