From a566ea7e8f5975b4d38c0092d7adb0acbda3a890 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 1 Sep 2026 20:37:09 -0700 Subject: [PATCH] [Perf] Avoid more h2d copies from non-pinned tensors (#54660) Signed-off-by: Nick Hill Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/models/qwen2_5_vl.py | 13 +-- vllm/model_executor/models/qwen2_vl.py | 3 +- .../models/transformers/multimodal.py | 5 +- vllm/multimodal/inputs.py | 88 ++++++++----------- vllm/v1/attention/backends/flashinfer.py | 27 +++++- 5 files changed, 72 insertions(+), 64 deletions(-) diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 035317116b..8b214f6acb 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -805,7 +805,7 @@ class Qwen2_5_VisionTransformer(nn.Module): window_index_thw, cu_seqlens_window_thw = self.get_window_index_thw(t, h, w) cos_thw, sin_thw = self.rotary_pos_emb_thw(t, h, w) - window_index_thw_dev = window_index_thw.to(cos_thw.device, non_blocking=True) + window_index_thw_dev = async_tensor_h2d(window_index_thw, cos_thw.device) cos_thw = cos_thw[window_index_thw_dev, :, :] cos_thw = cos_thw.flatten(start_dim=0, end_dim=1) sin_thw = sin_thw[window_index_thw_dev, :, :] @@ -1053,7 +1053,7 @@ class Qwen2_5_VisionTransformer(nn.Module): ) rotary_pos_emb_cos = rotary_pos_emb_cos.to(device=device, non_blocking=True) rotary_pos_emb_sin = rotary_pos_emb_sin.to(device=device, non_blocking=True) - window_index = window_index.to(device=device, non_blocking=True) + window_index = async_tensor_h2d(window_index, device) reverse_indices = reverse_indices.to(device=device, non_blocking=True) metadata["rotary_pos_emb_cos"] = rotary_pos_emb_cos @@ -1475,8 +1475,8 @@ class Qwen2_5_VLForConditionalGeneration( grid_thw_list = grid_thw.tolist() image_embeds_out = [] for emb, size in zip(image_embeds_split, grid_thw_list): - positions = compute_mrope_for_media(size, merge_size).to( - emb.device, non_blocking=True + positions = async_tensor_h2d( + compute_mrope_for_media(size, merge_size), emb.device ) emb = torch.cat([emb, positions], dim=1) image_embeds_out.append(emb) @@ -1558,12 +1558,13 @@ class Qwen2_5_VLForConditionalGeneration( spatial_merge_size=self.visual.spatial_merge_size, q=self.video_pruning_rate, ) - positions = compute_mrope_for_media( + positions_cpu = compute_mrope_for_media( size, merge_size, tokens_per_second=tokens_per_second, video_second_per_grid=video_second_per_grid_t.item(), - ).to(emb.device, non_blocking=True) + ) + positions = async_tensor_h2d(positions_cpu, emb.device) with gpu_sync_allowed(): emb = emb[retention_mask] diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index de75a59fee..2b0d674198 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -85,6 +85,7 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers @@ -707,7 +708,7 @@ class Qwen2VisionTransformer(nn.Module): return { "rotary_pos_emb_cos": rotary_pos_emb_cos, "rotary_pos_emb_sin": rotary_pos_emb_sin, - "cu_seqlens": cu_seqlens.to(device=device, non_blocking=True), + "cu_seqlens": async_tensor_h2d(cu_seqlens, device), "max_seqlen": max_seqlen, } diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index f4206a3e14..befc2ffffe 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -60,6 +60,7 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import async_tensor_h2d if TYPE_CHECKING: from transformers import BatchFeature, PreTrainedModel @@ -1180,8 +1181,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): # grid_thw fields are registered keep_on_cpu; restore the on-device # placement that HF get_image_features implementations expect. for key, value in kwargs.items(): - if isinstance(value, torch.Tensor): - kwargs[key] = value.to(pixel_values.device, non_blocking=True) + if isinstance(value, torch.Tensor) and value.is_cpu: + kwargs[key] = async_tensor_h2d(value, pixel_values.device) # The underlying HuggingFace `get_image_features` implementations # contain model-internal syncs (e.g. Idefics3 filters all-zero diff --git a/vllm/multimodal/inputs.py b/vllm/multimodal/inputs.py index c80a78893d..bd1fc169d1 100644 --- a/vllm/multimodal/inputs.py +++ b/vllm/multimodal/inputs.py @@ -293,21 +293,39 @@ def _nested_tensors_are_cpu(tensors: NestedTensors) -> bool: ) +def _is_dense(t: torch.Tensor) -> bool: + """Whether `t`'s elements fill a contiguous storage range, possibly + permuted (e.g. a transposed matrix). Such layouts can be copied with + pitched cudaMemcpy2D/3DAsync directly from pinned memory, whereas + gapped layouts (e.g. strided slices) stage through a pageable temp.""" + if not t.is_contiguous(): + expected_stride = 1 + for stride, size in sorted(zip(t.stride(), t.shape)): + if size > 1: + if stride != expected_stride: + return False + expected_stride *= size + return True + + def _nested_tensors_h2d( tensors: NestedTensors, device: torch.types.Device, + *, + pin_memory: bool = False, ) -> NestedTensors: if device is None: return tensors - return json_map_leaves( - ( - lambda x: x.to(device=device, non_blocking=True) - if isinstance(x, torch.Tensor) - else x - ), - tensors, - ) + def _h2d(x: torch.Tensor) -> torch.Tensor: + if not isinstance(x, torch.Tensor): + return x + if pin_memory and x.is_cpu and not (x.is_pinned() and _is_dense(x)): + # Ensure tensor is pinned and dense for non_blocking H2D copy. + x = x.new_empty(x.shape, pin_memory=True).copy_(x) + return x.to(device=device, non_blocking=True) + + return json_map_leaves(_h2d, tensors) BatchedTensorInputs: TypeAlias = dict[str, NestedTensors] @@ -452,10 +470,7 @@ class BaseMultiModalField(ABC): @abstractmethod def _reduce_data( - self, - batch: list[NestedTensors], - *, - pin_memory: bool, + self, batch: list[NestedTensors], *, pin_memory: bool ) -> NestedTensors: raise NotImplementedError @@ -491,7 +506,7 @@ class BaseMultiModalField(ABC): # concept, and `pin_memory()` rejects device tensors outright. pin_memory = False out = self._reduce_data(batch, pin_memory=pin_memory) - return _nested_tensors_h2d(out, device=device) + return _nested_tensors_h2d(out, device=device, pin_memory=pin_memory) @dataclass(frozen=True, kw_only=True) @@ -511,10 +526,7 @@ class MultiModalBatchedField(BaseMultiModalField): return [field_factory(item) for item in data] def _reduce_data( - self, - batch: list[NestedTensors], - *, - pin_memory: bool, + self, batch: list[NestedTensors], *, pin_memory: bool ) -> NestedTensors: if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"): batch = cast(list[torch.Tensor], batch) @@ -522,20 +534,11 @@ class MultiModalBatchedField(BaseMultiModalField): # An optimization when `batch` contains only one tensor: # - produce exactly same result as `torch.stack(batch)` # - will achieve zero-copy if the tensor is contiguous - out = batch[0].unsqueeze(0) - if not pin_memory: - return out.contiguous() - # Avoid extra copy - pinning unpinned memory will make it contiguous - if not out.is_contiguous() and out.is_pinned(): - out = out.contiguous() - return out.pin_memory() + return batch[0].unsqueeze(0) first_shape = batch[0].shape if all(elem.shape == first_shape for elem in batch): - out = torch.empty( - (len(batch), *batch[0].shape), - dtype=batch[0].dtype, - device=batch[0].device, - pin_memory=pin_memory, + out = batch[0].new_empty( + (len(batch), *first_shape), pin_memory=pin_memory ) return torch.stack(batch, out=out) @@ -567,10 +570,7 @@ class MultiModalFlatField(BaseMultiModalField): return [field_factory(data[cast(slice, s)]) for s in self.slices] def _reduce_data( - self, - batch: list[NestedTensors], - *, - pin_memory: bool, + self, batch: list[NestedTensors], *, pin_memory: bool ) -> NestedTensors: if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"): batch = cast(list[torch.Tensor], batch) @@ -578,13 +578,7 @@ class MultiModalFlatField(BaseMultiModalField): # An optimization when `batch` contains only one tensor: # - produce exactly same result as `torch.concat(batch)` # - will achieve zero-copy if the tensor is contiguous - out = batch[0] - if not pin_memory: - return out.contiguous() - # Avoid extra copy - pinning unpinned memory will make it contiguous - if not out.is_contiguous() and out.is_pinned(): - out = out.contiguous() - return out.pin_memory() + return batch[0] dim = self.dim + (self.dim < 0) * len(batch[0].shape) @@ -596,11 +590,8 @@ class MultiModalFlatField(BaseMultiModalField): if all(_shape_before_after(elem) == first_shape for elem in batch): shape_before, shape_after = first_shape shape_concat = sum(item.shape[dim] for item in batch) - out = torch.empty( - (*shape_before, shape_concat, *shape_after), - dtype=batch[0].dtype, - device=batch[0].device, - pin_memory=pin_memory, + out = batch[0].new_empty( + (*shape_before, shape_concat, *shape_after), pin_memory=pin_memory ) return torch.concat(batch, dim=self.dim, out=out) @@ -622,12 +613,7 @@ class MultiModalFlatField(BaseMultiModalField): max_sizes.append(max(t.shape[d] for t in batch)) # Step 2: Create zero-initialized output tensor - out = torch.zeros( - max_sizes, - dtype=batch[0].dtype, - device=batch[0].device, - pin_memory=pin_memory, - ) + out = batch[0].new_zeros(max_sizes, pin_memory=pin_memory) # Step 3: Slice-assign each tensor to its proper position concat_offset = 0 diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index ecbf3a84e7..07c19ea6b9 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -1569,6 +1569,17 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): prefill_start : num_reqs + 1 ] assert paged_kv_indptr_prefill_cpu.shape[0] == num_prefills + 1 + # plan() copies these to the GPU with non_blocking=True; + # stage them in pinned memory so the copies stay async + # (the reused buffers themselves are intentionally not pinned). + if PIN_MEMORY: + qo_indptr_prefill_cpu = qo_indptr_prefill_cpu.pin_memory() + paged_kv_indptr_prefill_cpu = ( + paged_kv_indptr_prefill_cpu.pin_memory() + ) + paged_kv_last_page_len_prefill_cpu = ( + paged_kv_last_page_len_prefill_cpu.pin_memory() + ) if self.use_dcp: assert isinstance(prefill_wrapper, BatchDCPPrefillWrapper) prefill_wrapper.plan( @@ -1695,13 +1706,21 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): o_dtype = ( FP8_DTYPE if self.is_kvcache_nvfp4 else self.model_config.dtype ) + paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_input_tokens + 1] + paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[ + :num_input_tokens + ] + # plan() copies these to the GPU with non_blocking=True; + # stage them in pinned memory so the copies stay async + # (the reused buffers themselves are intentionally not pinned). + if PIN_MEMORY: + paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory() + paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory() fast_plan_decode( decode_wrapper, - indptr_cpu=self.paged_kv_indptr.cpu[: num_input_tokens + 1], + indptr_cpu=paged_kv_indptr_cpu, indices=paged_kv_indices, - last_page_len_cpu=self.paged_kv_last_page_len.cpu[ - :num_input_tokens - ], + last_page_len_cpu=paged_kv_last_page_len_cpu, num_qo_heads=self.num_qo_heads * self.dcp_world_size, num_kv_heads=self.num_kv_heads, head_dim=self.head_dim,