diff --git a/vllm/model_executor/layers/pooler/tokwise/methods.py b/vllm/model_executor/layers/pooler/tokwise/methods.py index 1eb8a8ba3d..a6a1bfff1a 100644 --- a/vllm/model_executor/layers/pooler/tokwise/methods.py +++ b/vllm/model_executor/layers/pooler/tokwise/methods.py @@ -11,6 +11,7 @@ from vllm.config import get_current_vllm_config from vllm.config.pooler import TokenPoolingType from vllm.model_executor.layers.pooler import PoolingParamsUpdate from vllm.tasks import PoolingTask +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.pool.metadata import PoolingMetadata TokenPoolingMethodOutputItem: TypeAlias = torch.Tensor | None @@ -131,7 +132,7 @@ class StepPool(AllPool): if step_tag_id is not None: idx_cpu = (token_id_cpu == step_tag_id).nonzero(as_tuple=True)[0] - idx = idx_cpu.to(data.device, non_blocking=True) + idx = async_tensor_h2d(idx_cpu, data.device) data = data[idx] pooled_data.append(data) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index 8fbd31ab4e..e904f07fc4 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -158,13 +158,13 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device) yield module return - original_device_states: dict[str, torch.device] = {} + cpu_params: set[str] = set() uva_offloaded_parameters: list[str] = [] - # Store original device states and move parameters to GPU if they're on CPU + # Store which parameters are on CPU and move them to the GPU for name, p in module.named_parameters(): if p.device.type == "cpu": - original_device_states[name] = p.device + cpu_params.add(name) p.data = p.data.to(target_device) if getattr(p, "_vllm_is_uva_offloaded", False): uva_offloaded_parameters.append(name) @@ -178,20 +178,21 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device) is_pin_memory_available() and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY ) - # Restore parameters to their original devices, ignoring new parameters + # Restore the CPU-resident parameters, ignoring new parameters. for name, p in module.named_parameters(): - if name in original_device_states: - original_device: torch.device = original_device_states[name] - p.data = p.data.to(original_device) + if name in cpu_params: + p.data = torch.empty_like( + p.data, device="cpu", pin_memory=use_pin_memory + ).copy_(p.data) # parameter is UVA offloaded, but was replaced with a new device tensor # re-offload it to CPU using UVA if name in uva_offloaded_parameters and not getattr( p, "_vllm_is_uva_offloaded", False ): - cpu_data = p.data.to(device="cpu") - if use_pin_memory: - cpu_data = cpu_data.pin_memory() + cpu_data = torch.empty_like( + p.data, device="cpu", pin_memory=use_pin_memory + ).copy_(p.data) p.data = get_accelerator_view_from_cpu_tensor(cpu_data) p._vllm_is_uva_offloaded = True diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index ace449b25b..06a68f62f3 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -425,10 +425,11 @@ class Ernie4_5_VisionTransformer(nn.Module): return self.patch_embed.proj.weight.device def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor: + device = self.device pos_ids = [] for t, h, w in grid_thw: - hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) - wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w) + wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1) hpos_ids = ( hpos_ids.reshape( h // self.spatial_merge_size, @@ -453,9 +454,6 @@ class Ernie4_5_VisionTransformer(nn.Module): pos_ids = torch.cat(pos_ids, dim=0) max_grid_size = grid_thw[:, 1:].max() rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) - # `pos_ids` is built on the host; stage it over non-blocking so the - # gather below doesn't index a device tensor with a CPU one. - pos_ids = pos_ids.to(rotary_pos_emb_full.device, non_blocking=True) rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 96bed8b6ee..ec72210a10 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -709,10 +709,11 @@ class Glm4vVisionTransformer(nn.Module): def rot_pos_emb( self, grid_thw: list[list[int]] ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + device = self.device pos_ids = [] for t, h, w in grid_thw: - hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) - wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w) + wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1) hpos_ids = ( hpos_ids.reshape( h // self.spatial_merge_size, @@ -740,7 +741,6 @@ class Glm4vVisionTransformer(nn.Module): # Use pre-computed cos_sin_cache from RotaryEmbedding cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size) - pos_ids = pos_ids.to(cos.device, non_blocking=True) cos_combined = cos[pos_ids].flatten(1) sin_combined = sin[pos_ids].flatten(1) return cos_combined, sin_combined, pos_ids diff --git a/vllm/model_executor/models/idefics2_vision_model.py b/vllm/model_executor/models/idefics2_vision_model.py index 09887c1ec3..86b05f7478 100644 --- a/vllm/model_executor/models/idefics2_vision_model.py +++ b/vllm/model_executor/models/idefics2_vision_model.py @@ -39,6 +39,7 @@ from vllm.model_executor.layers.linear import ( ) from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import PIN_MEMORY from .utils import AutoWeightsLoader, WeightsMapper from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model @@ -85,7 +86,9 @@ class Idefics2VisionEmbeddings(nn.Module): 1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side ) position_ids = torch.full( - size=(batch_size, max_nb_patches_h * max_nb_patches_w), fill_value=0 + size=(batch_size, max_nb_patches_h * max_nb_patches_w), + fill_value=0, + pin_memory=PIN_MEMORY, ) with gpu_sync_allowed(): diff --git a/vllm/model_executor/models/phi4mm_audio.py b/vllm/model_executor/models/phi4mm_audio.py index cd97522705..b043341ff0 100644 --- a/vllm/model_executor/models/phi4mm_audio.py +++ b/vllm/model_executor/models/phi4mm_audio.py @@ -32,6 +32,7 @@ from vllm.model_executor.models.phi4mm_utils import ( get_offset, unfold_tensor, ) +from vllm.utils.torch_utils import async_tensor_h2d class ConformerEncoderLayer(nn.Module): @@ -586,9 +587,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module): seq_len, batch_size, self.chunk_size, self.left_chunk ) device = xs_pad.device - enc_streaming_mask = enc_streaming_mask.contiguous().to( - device, non_blocking=True - ) + enc_streaming_mask = async_tensor_h2d(enc_streaming_mask.contiguous(), device) xs_pad = xs_pad.to(device) input_tensor = xs_pad @@ -921,9 +920,7 @@ class ConformerEncoder(TransformerEncoderBase): enc_streaming_mask = self._streaming_mask( max_audio_length, batch_size, self.chunk_size, self.left_chunk ) - enc_streaming_mask = enc_streaming_mask.contiguous().to( - device, non_blocking=True - ) + enc_streaming_mask = async_tensor_h2d(enc_streaming_mask.contiguous(), device) if mask is None: return enc_streaming_mask diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 3e5deaad91..e8534c2abb 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -715,8 +715,11 @@ class Qwen2_5_VisionTransformer(nn.Module): return self.patch_embed.proj.weight.device def rotary_pos_emb_thw(self, t, h, w): - hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) - wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + # Use pre-computed cos_sin_cache from RotaryEmbedding + cos, sin = self.rotary_pos_emb.get_cos_sin(max(h, w)) + + hpos_ids = torch.arange(h, device=cos.device).unsqueeze(1).expand(-1, w) + wpos_ids = torch.arange(w, device=cos.device).unsqueeze(0).expand(h, -1) hpos_ids = ( hpos_ids.reshape( h // self.spatial_merge_size, @@ -738,12 +741,7 @@ class Qwen2_5_VisionTransformer(nn.Module): .flatten() ) pos_ids = torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1) - max_size = max(h, w) - # Use pre-computed cos_sin_cache from RotaryEmbedding - cos, sin = self.rotary_pos_emb.get_cos_sin(max_size) - - pos_ids = pos_ids.to(cos.device, non_blocking=True) cos_combined = cos[pos_ids].flatten(1) sin_combined = sin[pos_ids].flatten(1) diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 700fefe585..e984e94a64 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -609,11 +609,12 @@ class Qwen2VisionTransformer(nn.Module): def rot_pos_emb( self, grid_thw: list[list[int]] ) -> tuple[torch.Tensor, torch.Tensor]: + device = self.device pos_ids = [] max_grid_size = 0 for t, h, w in grid_thw: - hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) - wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w) + wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1) hpos_ids = ( hpos_ids.reshape( h // self.spatial_merge_size, @@ -641,7 +642,6 @@ class Qwen2VisionTransformer(nn.Module): # Use pre-computed cos_sin_cache from RotaryEmbedding cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size) - pos_ids = pos_ids.to(cos.device, non_blocking=True) cos_combined = cos[pos_ids].flatten(1) sin_combined = sin[pos_ids].flatten(1) return cos_combined, sin_combined diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index 7cea97ef5b..921d14dd13 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -94,6 +94,7 @@ from vllm.transformers_utils.processors.qwen3_asr import ( Qwen3ASRProcessor, ) from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import async_tensor_h2d logger = init_logger(__name__) _ASR_TEXT_TAG = "" @@ -465,8 +466,8 @@ class Qwen3ASRForConditionalGeneration( input_features = audio_input["input_features"] # audio_feature_lengths is keep_on_cpu; the audio tower derives # device placement from feature_lens, so move it explicitly. - audio_feature_lengths = audio_input["audio_feature_lengths"].to( - input_features.device, non_blocking=True + audio_feature_lengths = async_tensor_h2d( + audio_input["audio_feature_lengths"], input_features.device ) audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths) diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index 9f309d27f5..f2bc7aa965 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -842,9 +842,14 @@ class Qwen3Omni_VisionTransformer(nn.Module): return self.patch_embed.proj.weight.device def rot_pos_emb(self, grid_thw): + max_grid_size = grid_thw[:, 1:].max() + + # Use pre-computed cos_sin_cache from RotaryEmbedding + cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size) + pos_ids = [] for t, h, w in grid_thw: - hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + hpos_ids = torch.arange(h, device=cos.device).unsqueeze(1).expand(-1, w) hpos_ids = hpos_ids.reshape( h // self.spatial_merge_size, self.spatial_merge_size, @@ -854,7 +859,7 @@ class Qwen3Omni_VisionTransformer(nn.Module): hpos_ids = hpos_ids.permute(0, 2, 1, 3) hpos_ids = hpos_ids.flatten() - wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + wpos_ids = torch.arange(w, device=cos.device).unsqueeze(0).expand(h, -1) wpos_ids = wpos_ids.reshape( h // self.spatial_merge_size, self.spatial_merge_size, @@ -865,12 +870,7 @@ class Qwen3Omni_VisionTransformer(nn.Module): wpos_ids = wpos_ids.flatten() pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) - max_grid_size = grid_thw[:, 1:].max() - # Use pre-computed cos_sin_cache from RotaryEmbedding - cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size) - - pos_ids = pos_ids.to(cos.device, non_blocking=True) cos_combined = cos[pos_ids].flatten(1) sin_combined = sin[pos_ids].flatten(1) diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index cf325b64f9..c5625b217f 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -56,6 +56,7 @@ from vllm.multimodal.processing import ( from vllm.sequence import IntermediateTensors from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.tensor_schema import TensorSchema, TensorShape +from vllm.utils.torch_utils import async_tensor_h2d from .clip import dual_encoder_has_text_tokens, merge_dual_encoder_text_and_vision from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant @@ -1010,7 +1011,7 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant): flip_indices_cpu = ( starts_cpu[sequence_ids_cpu] + ends_cpu[sequence_ids_cpu] ) - (1 + current_positions_cpu) - flip_indices = flip_indices_cpu.to(features.device, non_blocking=True) + flip_indices = async_tensor_h2d(flip_indices_cpu, features.device) return features[flip_indices] diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index a479b2ca48..ecbf3a84e7 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -917,16 +917,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): arch, ) # Preparing persistent buffers - # Since we do not have explicit synchronization in ModelRunnerV2, we do not pin - # reused CPU buffers to avoid a race condition between step N async copies to - # GPU and step N+1 buffer updates. - self.pin_memory = not vllm_config.use_v2_model_runner and PIN_MEMORY - self.paged_kv_indptr = self._make_buffer(max_num_reqs + 1) - self.paged_kv_indptr_cpu_buffer = torch.zeros_like( - self.paged_kv_indptr.cpu, pin_memory=self.pin_memory - ) # Extra buffer for mutable paged_kv_indptr.cpu in cuda graph mode - self.paged_kv_indices = self._make_buffer(max_num_pages) - self.paged_kv_last_page_len = self._make_buffer(max_num_reqs) + self.paged_kv_indptr = CpuGpuBuffer( + max_num_reqs + 1, dtype=torch.int32, device=self.device, pin_memory=False + ) + self.paged_kv_indices = torch.zeros( + max_num_pages, dtype=torch.int32, device=self.device + ) + self.paged_kv_last_page_len = CpuGpuBuffer( + max_num_reqs, dtype=torch.int32, device=self.device, pin_memory=False + ) @property def kv_cache_layout(self) -> KVCacheLayout: @@ -970,17 +969,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3") return self.kv_cache_spec.dtype - def _make_buffer( - self, *size: int | torch.SymInt, dtype: torch.dtype = torch.int32 - ) -> CpuGpuBuffer: - return CpuGpuBuffer( - *size, - dtype=dtype, - device=self.device, - pin_memory=self.pin_memory, - with_numpy=True, - ) - @override # type: ignore[misc] @classmethod def get_cudagraph_support( @@ -1197,7 +1185,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): if decode_wrapper is None: if use_cudagraph: paged_kv_indptr = self.paged_kv_indptr.gpu[: batch_size + 1] - paged_kv_indices = self.paged_kv_indices.gpu + paged_kv_indices = self.paged_kv_indices paged_kv_last_page_len = self.paged_kv_last_page_len.gpu[:batch_size] else: paged_kv_indptr = None @@ -1260,20 +1248,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): dtype=np.int32, out=self.paged_kv_indptr.np[1 : num_reqs + 1], ) - # NOTE(woosuk): Because self.paged_kv_indptr_cpu can be modified - # after this line (e.g., for cuda graphs), we need to copy the data to - # self.paged_kv_indptr_buffer to avoid race condition. - self.paged_kv_indptr_cpu_buffer[: num_reqs + 1] = self.paged_kv_indptr.cpu[ - : num_reqs + 1 - ] paged_kv_indptr = self.paged_kv_indptr.gpu[: num_reqs + 1] - paged_kv_indptr.copy_( - self.paged_kv_indptr_cpu_buffer[: num_reqs + 1], non_blocking=True - ) + paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_reqs + 1] + if PIN_MEMORY: + paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory() + paged_kv_indptr.copy_(paged_kv_indptr_cpu, non_blocking=True) # write self.paged_kv_indices inplace num_actual_pages = self.paged_kv_indptr.np[num_reqs] - paged_kv_indices = self.paged_kv_indices.gpu[:num_actual_pages] + paged_kv_indices = self.paged_kv_indices[:num_actual_pages] _copy_page_indices_kernel[(num_reqs,)]( paged_kv_indices, block_table_tensor, @@ -1289,8 +1272,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): page_size, paged_kv_last_page_len_np, ) + paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[:num_reqs] + if PIN_MEMORY: + paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory() self.paged_kv_last_page_len.gpu[:num_reqs].copy_( - self.paged_kv_last_page_len.cpu[:num_reqs], non_blocking=True + paged_kv_last_page_len_cpu, non_blocking=True ) return paged_kv_indices diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 94f1776c05..4e89688503 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -11,6 +11,7 @@ import numpy as np import torch from vllm.compilation.cuda_graph import CUDAGraphStat +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.core.sched.output import SchedulerOutput if TYPE_CHECKING: @@ -182,11 +183,18 @@ class LogprobsTensors(NamedTuple): """Create empty LogprobsTensors on CPU.""" logprob_token_ids = torch.empty( - (num_positions, num_tokens_per_position), dtype=torch.int32, device="cpu" + (num_positions, num_tokens_per_position), + dtype=torch.int32, + device="cpu", + pin_memory=PIN_MEMORY, ) - logprobs = torch.empty_like(logprob_token_ids, dtype=torch.float32) - selected_token_ranks = torch.empty( - num_positions, dtype=torch.int32, device="cpu" + logprobs = logprob_token_ids.new_empty( + (num_positions, num_tokens_per_position), + dtype=torch.float32, + pin_memory=PIN_MEMORY, + ) + selected_token_ranks = logprob_token_ids.new_empty( + num_positions, pin_memory=PIN_MEMORY ) return LogprobsTensors( logprob_token_ids=logprob_token_ids, diff --git a/vllm/v1/worker/gpu/mm/encoder_runner.py b/vllm/v1/worker/gpu/mm/encoder_runner.py index 49b4adee18..16e42d140a 100644 --- a/vllm/v1/worker/gpu/mm/encoder_runner.py +++ b/vllm/v1/worker/gpu/mm/encoder_runner.py @@ -215,7 +215,10 @@ class EncoderRunner: num_computed_tokens = num_computed_tokens + draft_lookahead is_mm_embed = torch.zeros( - total_num_scheduled_tokens, dtype=torch.bool, device="cpu" + total_num_scheduled_tokens, + dtype=torch.bool, + device="cpu", + pin_memory=PIN_MEMORY, ) # Whether to gather media embeddings this step.