[Perf] Avoid more h2d copies from non-pinned tensors (#54660)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3ba9907a1d
commit
a566ea7e8f
@@ -805,7 +805,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
window_index_thw, cu_seqlens_window_thw = self.get_window_index_thw(t, h, w)
|
||||
cos_thw, sin_thw = self.rotary_pos_emb_thw(t, h, w)
|
||||
|
||||
window_index_thw_dev = window_index_thw.to(cos_thw.device, non_blocking=True)
|
||||
window_index_thw_dev = async_tensor_h2d(window_index_thw, cos_thw.device)
|
||||
cos_thw = cos_thw[window_index_thw_dev, :, :]
|
||||
cos_thw = cos_thw.flatten(start_dim=0, end_dim=1)
|
||||
sin_thw = sin_thw[window_index_thw_dev, :, :]
|
||||
@@ -1053,7 +1053,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
)
|
||||
rotary_pos_emb_cos = rotary_pos_emb_cos.to(device=device, non_blocking=True)
|
||||
rotary_pos_emb_sin = rotary_pos_emb_sin.to(device=device, non_blocking=True)
|
||||
window_index = window_index.to(device=device, non_blocking=True)
|
||||
window_index = async_tensor_h2d(window_index, device)
|
||||
reverse_indices = reverse_indices.to(device=device, non_blocking=True)
|
||||
|
||||
metadata["rotary_pos_emb_cos"] = rotary_pos_emb_cos
|
||||
@@ -1475,8 +1475,8 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
grid_thw_list = grid_thw.tolist()
|
||||
image_embeds_out = []
|
||||
for emb, size in zip(image_embeds_split, grid_thw_list):
|
||||
positions = compute_mrope_for_media(size, merge_size).to(
|
||||
emb.device, non_blocking=True
|
||||
positions = async_tensor_h2d(
|
||||
compute_mrope_for_media(size, merge_size), emb.device
|
||||
)
|
||||
emb = torch.cat([emb, positions], dim=1)
|
||||
image_embeds_out.append(emb)
|
||||
@@ -1558,12 +1558,13 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
spatial_merge_size=self.visual.spatial_merge_size,
|
||||
q=self.video_pruning_rate,
|
||||
)
|
||||
positions = compute_mrope_for_media(
|
||||
positions_cpu = compute_mrope_for_media(
|
||||
size,
|
||||
merge_size,
|
||||
tokens_per_second=tokens_per_second,
|
||||
video_second_per_grid=video_second_per_grid_t.item(),
|
||||
).to(emb.device, non_blocking=True)
|
||||
)
|
||||
positions = async_tensor_h2d(positions_cpu, emb.device)
|
||||
|
||||
with gpu_sync_allowed():
|
||||
emb = emb[retention_mask]
|
||||
|
||||
@@ -85,6 +85,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||
|
||||
@@ -707,7 +708,7 @@ class Qwen2VisionTransformer(nn.Module):
|
||||
return {
|
||||
"rotary_pos_emb_cos": rotary_pos_emb_cos,
|
||||
"rotary_pos_emb_sin": rotary_pos_emb_sin,
|
||||
"cu_seqlens": cu_seqlens.to(device=device, non_blocking=True),
|
||||
"cu_seqlens": async_tensor_h2d(cu_seqlens, device),
|
||||
"max_seqlen": max_seqlen,
|
||||
}
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import BatchFeature, PreTrainedModel
|
||||
@@ -1180,8 +1181,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
# grid_thw fields are registered keep_on_cpu; restore the on-device
|
||||
# placement that HF get_image_features implementations expect.
|
||||
for key, value in kwargs.items():
|
||||
if isinstance(value, torch.Tensor):
|
||||
kwargs[key] = value.to(pixel_values.device, non_blocking=True)
|
||||
if isinstance(value, torch.Tensor) and value.is_cpu:
|
||||
kwargs[key] = async_tensor_h2d(value, pixel_values.device)
|
||||
|
||||
# The underlying HuggingFace `get_image_features` implementations
|
||||
# contain model-internal syncs (e.g. Idefics3 filters all-zero
|
||||
|
||||
Reference in New Issue
Block a user