From 7df3d7dada840c68b85b26b79de7f59f676d58e3 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Sat, 20 Jun 2026 20:02:24 -0700 Subject: [PATCH] [Core] Ensure memory is pinned prior to async h2d copy (#45424) Signed-off-by: Nick Hill --- .../v1/logits_processors/test_correctness.py | 2 - .../test_gpu_model_runner_streaming.py | 1 - tests/v1/worker/test_gpu_input_batch.py | 6 -- tests/v1/worker/test_gpu_model_runner.py | 4 - vllm/device_allocator/cumem.py | 4 +- vllm/device_allocator/xpumem.py | 4 +- vllm/lora/lora_model.py | 4 +- vllm/lora/lora_weights.py | 4 +- vllm/lora/model_manager.py | 4 +- .../layers/attention/mla_attention.py | 18 ++-- .../layers/attention/mm_encoder_attention.py | 3 +- .../layers/pooler/seqwise/methods.py | 16 ++-- vllm/model_executor/models/moonvit.py | 5 +- vllm/model_executor/models/qwen2_5_vl.py | 5 +- vllm/models/deepseek_v4/sparse_mla.py | 3 +- vllm/multimodal/inputs.py | 16 +++- vllm/platforms/__init__.py | 3 +- vllm/platforms/cuda.py | 3 +- vllm/platforms/xpu.py | 12 +-- vllm/utils/torch_utils.py | 33 ++++---- vllm/v1/attention/backends/flashinfer.py | 6 +- vllm/v1/attention/backends/flex_attention.py | 8 +- vllm/v1/attention/backends/gdn_attn.py | 19 +++-- vllm/v1/attention/backends/mamba2_attn.py | 27 +++--- .../backends/mla/flashinfer_mla_sparse.py | 4 +- .../attention/backends/mla/flashmla_sparse.py | 4 +- vllm/v1/attention/backends/utils.py | 35 ++++---- vllm/v1/kv_offload/cpu/gpu_worker.py | 6 +- vllm/v1/pool/metadata.py | 6 +- vllm/v1/sample/logits_processor/builtin.py | 15 ++-- vllm/v1/sample/ops/penalties.py | 5 +- vllm/v1/sample/sampler.py | 4 +- vllm/v1/sample/thinking_budget_state.py | 5 +- vllm/v1/serial_utils.py | 4 +- vllm/v1/simple_kv_offload/worker.py | 4 +- vllm/v1/spec_decode/extract_hidden_states.py | 5 +- vllm/v1/spec_decode/llm_base_proposer.py | 16 ++-- vllm/v1/spec_decode/ngram_proposer_gpu.py | 4 +- .../backend_lm_format_enforcer.py | 4 +- vllm/v1/structured_output/backend_outlines.py | 4 +- vllm/v1/structured_output/utils.py | 25 +++--- vllm/v1/utils.py | 3 +- vllm/v1/worker/cpu/shm.py | 11 ++- vllm/v1/worker/gpu/buffer_utils.py | 11 ++- vllm/v1/worker/gpu/mm/encoder_runner.py | 5 +- vllm/v1/worker/gpu/model_runner.py | 5 +- vllm/v1/worker/gpu/model_states/whisper.py | 7 +- vllm/v1/worker/gpu_input_batch.py | 33 ++++---- vllm/v1/worker/gpu_model_runner.py | 83 +++++++++---------- 49 files changed, 254 insertions(+), 264 deletions(-) diff --git a/tests/v1/logits_processors/test_correctness.py b/tests/v1/logits_processors/test_correctness.py index 80083fd57f..c93593865e 100644 --- a/tests/v1/logits_processors/test_correctness.py +++ b/tests/v1/logits_processors/test_correctness.py @@ -145,7 +145,6 @@ def _generate_fake_sampling_metadata( vllm_config.scheduler_config.max_num_seqs, num_spec, device, - PIN_MEMORY_AVAILABLE, ) fake_sampling_metadata = SamplingMetadata( temperature=torch.full((batch_size,), 0.0), @@ -880,7 +879,6 @@ def test_maybe_create_thinking_budget_holder_without_reasoning(): cfg.scheduler_config.max_num_seqs, 0, torch.device("cpu"), - False, ) is None ) diff --git a/tests/v1/streaming_input/test_gpu_model_runner_streaming.py b/tests/v1/streaming_input/test_gpu_model_runner_streaming.py index 946ca99507..9b130e570f 100644 --- a/tests/v1/streaming_input/test_gpu_model_runner_streaming.py +++ b/tests/v1/streaming_input/test_gpu_model_runner_streaming.py @@ -35,7 +35,6 @@ def mock_model_runner_with_input_batch(): max_model_len=1024, max_num_batched_tokens=1024, device="cpu", - pin_memory=False, vocab_size=32000, block_sizes=[16], kernel_block_sizes=[16], diff --git a/tests/v1/worker/test_gpu_input_batch.py b/tests/v1/worker/test_gpu_input_batch.py index 3a478d2101..bfd4016c9f 100644 --- a/tests/v1/worker/test_gpu_input_batch.py +++ b/tests/v1/worker/test_gpu_input_batch.py @@ -10,7 +10,6 @@ import torch from vllm.platforms import current_platform from vllm.sampling_params import SamplingParams -from vllm.utils.platform_utils import is_pin_memory_available from vllm.utils.torch_utils import make_tensor_with_pad from vllm.v1.pool.metadata import PoolingMetadata from vllm.v1.sample.logits_processor import LogitsProcessors @@ -236,7 +235,6 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int): max_model_len=1024, max_num_batched_tokens=1024, device=torch.device(device), - pin_memory=is_pin_memory_available(), vocab_size=1024, block_sizes=[1], kernel_block_sizes=[1], @@ -331,7 +329,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis max_model_len=1024, max_num_batched_tokens=1024, device=torch.device(device), - pin_memory=is_pin_memory_available(), vocab_size=1024, block_sizes=[1], kernel_block_sizes=[1], @@ -341,7 +338,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis max_model_len=1024, max_num_batched_tokens=1024, device=torch.device(device), - pin_memory=is_pin_memory_available(), vocab_size=1024, block_sizes=[1], kernel_block_sizes=[1], @@ -410,7 +406,6 @@ def test_pooling_prompt_lens_not_aliased(device: str): max_model_len=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS, max_num_batched_tokens=batch_size * (MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS), device=torch.device(device), - pin_memory=is_pin_memory_available(), vocab_size=VOCAB_SIZE, block_sizes=[16], kernel_block_sizes=[16], @@ -459,7 +454,6 @@ def test_pooling_metadata_token_id_buffers( max_model_len=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS, max_num_batched_tokens=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS, device=torch.device("cpu"), - pin_memory=False, vocab_size=VOCAB_SIZE, block_sizes=[16], kernel_block_sizes=[16], diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 80dd8ee306..75d8c9c746 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -85,7 +85,6 @@ def initialize_kv_cache(runner: GPUModelRunner): max_model_len=runner.max_model_len, max_num_batched_tokens=runner.max_num_tokens, device=runner.device, - pin_memory=runner.pin_memory, vocab_size=runner.model_config.get_vocab_size(), block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size], kernel_block_sizes=[ @@ -1405,7 +1404,6 @@ def test_input_batch_with_kernel_block_sizes(): max_model_len = 512 max_num_batched_tokens = 512 device = torch.device(DEVICE_TYPE) - pin_memory = False vocab_size = 50272 # Test with different kernel block sizes @@ -1417,7 +1415,6 @@ def test_input_batch_with_kernel_block_sizes(): max_model_len=max_model_len, max_num_batched_tokens=max_num_batched_tokens, device=device, - pin_memory=pin_memory, vocab_size=vocab_size, block_sizes=block_sizes, kernel_block_sizes=kernel_block_sizes, @@ -1478,7 +1475,6 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init): max_model_len=runner.max_model_len, max_num_batched_tokens=runner.max_num_tokens, device=runner.device, - pin_memory=runner.pin_memory, vocab_size=runner.model_config.get_vocab_size(), block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size], kernel_block_sizes=[16], diff --git a/vllm/device_allocator/cumem.py b/vllm/device_allocator/cumem.py index c30790df9c..59c0cf45f5 100644 --- a/vllm/device_allocator/cumem.py +++ b/vllm/device_allocator/cumem.py @@ -18,8 +18,8 @@ import torch from vllm.device_allocator import AllocationData, HandleType from vllm.logger import init_logger -from vllm.utils.platform_utils import is_pin_memory_available from vllm.utils.system_utils import find_loaded_library +from vllm.utils.torch_utils import PIN_MEMORY logger = init_logger(__name__) @@ -196,7 +196,7 @@ class CuMemAllocator: size_in_bytes, dtype=torch.uint8, device="cpu", - pin_memory=is_pin_memory_available(), + pin_memory=PIN_MEMORY, ) cpu_ptr = cpu_backup_tensor.data_ptr() libcudart.cudaMemcpy(cpu_ptr, ptr, size_in_bytes) diff --git a/vllm/device_allocator/xpumem.py b/vllm/device_allocator/xpumem.py index 7d99ced7ef..e0f359b200 100644 --- a/vllm/device_allocator/xpumem.py +++ b/vllm/device_allocator/xpumem.py @@ -11,7 +11,7 @@ import torch from vllm.device_allocator import AllocationData, HandleType from vllm.logger import init_logger -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY logger = init_logger(__name__) @@ -188,7 +188,7 @@ class XpuMemAllocator: size_in_bytes, dtype=torch.uint8, device="cpu", - pin_memory=is_pin_memory_available(), + pin_memory=PIN_MEMORY, ) cpu_ptr = cpu_backup_tensor.data_ptr() _xpu_memcpy_sync( diff --git a/vllm/lora/lora_model.py b/vllm/lora/lora_model.py index e3cb82e356..859ed02f87 100644 --- a/vllm/lora/lora_model.py +++ b/vllm/lora/lora_model.py @@ -17,7 +17,7 @@ from vllm.lora.utils import ( ) from vllm.model_executor.model_loader.tensorizer import TensorizerConfig from vllm.model_executor.models.utils import WeightsMapper -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY logger = init_logger(__name__) @@ -126,7 +126,7 @@ class LoRAModel: skip_prefixes: list[str] | None = None, ) -> "LoRAModel": """Create a LoRAModel from a dictionary of tensors.""" - pin_memory = str(device) == "cpu" and is_pin_memory_available() + pin_memory = str(device) == "cpu" and PIN_MEMORY loras: dict[str, LoRALayerWeights] = {} for tensor_name, tensor in tensors.items(): if is_base_embedding_weights(tensor_name): diff --git a/vllm/lora/lora_weights.py b/vllm/lora/lora_weights.py index 90b7df818a..f90724c5eb 100644 --- a/vllm/lora/lora_weights.py +++ b/vllm/lora/lora_weights.py @@ -7,7 +7,7 @@ import torch import torch.types from vllm.lora.peft_helper import PEFTHelper -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY class LoRALayerWeights: @@ -79,7 +79,7 @@ class LoRALayerWeights: dtype: torch.dtype, device: torch.types.Device, ) -> "LoRALayerWeights": - pin_memory = str(device) == "cpu" and is_pin_memory_available() + pin_memory = str(device) == "cpu" and PIN_MEMORY lora_a = torch.zeros( [rank, input_dim], dtype=dtype, device=device, pin_memory=pin_memory ) diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index 8063f485bf..39c3bb0ea1 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -42,7 +42,7 @@ from vllm.model_executor.models.utils import PPMissingLayer from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.encoder_budget import MultiModalBudget from vllm.utils.cache import LRUCache -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY logger = init_logger(__name__) @@ -801,7 +801,7 @@ class LoRAModelManager: # 2. The weight packing above (e.g., pack_moe) may invalidate the # pin_memory allocation, so we execute it after packing. - pin_memory = str(lora_device) == "cpu" and is_pin_memory_available() + pin_memory = str(lora_device) == "cpu" and PIN_MEMORY if pin_memory: for lora in lora_model.loras.values(): if isinstance(lora.lora_a, list): diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index ab3874c5da..247d6dc3a4 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -1684,12 +1684,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): # [[0, 0, 0, 0], [256, 256, 256, 256], [512, 512, 512, 512]] # Note(simon): this is done in CPU because of downstream's # of `to_list`. - chunk_starts = ( + chunk_starts = torch.empty( + num_chunks, num_prefills, dtype=torch.int32, pin_memory=True + ).copy_( torch.arange(num_chunks, dtype=torch.int32) + .multiply_(max_context_chunk) .unsqueeze(1) - .expand(-1, num_prefills) - * max_context_chunk - ).pin_memory() + ) chunk_ends = torch.min( context_lens_cpu.unsqueeze(0), chunk_starts + max_context_chunk ) @@ -1746,12 +1747,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): ) * self.dcp_local_block_size ) - local_chunk_starts = ( + local_chunk_starts = torch.empty( + num_chunks, num_prefills, dtype=torch.int32, pin_memory=True + ).copy_( torch.arange(num_chunks, dtype=torch.int32) + .multiply_(padded_local_max_context_chunk_across_ranks) .unsqueeze(1) - .expand(-1, num_prefills) - * padded_local_max_context_chunk_across_ranks - ).pin_memory() + ) local_chunk_ends = torch.min( padded_local_context_lens_cpu.unsqueeze(0), local_chunk_starts diff --git a/vllm/model_executor/layers/attention/mm_encoder_attention.py b/vllm/model_executor/layers/attention/mm_encoder_attention.py index 2ca051ad9e..bb1c995aeb 100644 --- a/vllm/model_executor/layers/attention/mm_encoder_attention.py +++ b/vllm/model_executor/layers/attention/mm_encoder_attention.py @@ -28,6 +28,7 @@ from vllm.utils.flashinfer import ( is_flashinfer_cudnn_fp8_prefill_attn_supported, ) from vllm.utils.math_utils import round_up +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backends.fa_utils import get_flash_attn_version from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.ops.vit_attn_wrappers import ( @@ -311,7 +312,7 @@ class MMEncoderAttention(CustomOp): ) cu_seqlens = np.concatenate([cu_seqlens_qko, cu_seqlens_v]) - cu_seqlens = torch.from_numpy(cu_seqlens).to(device, non_blocking=True) + cu_seqlens = async_tensor_h2d(cu_seqlens, device=device) return cu_seqlens def __init__( diff --git a/vllm/model_executor/layers/pooler/seqwise/methods.py b/vllm/model_executor/layers/pooler/seqwise/methods.py index 06dddde7de..5dea5d7627 100644 --- a/vllm/model_executor/layers/pooler/seqwise/methods.py +++ b/vllm/model_executor/layers/pooler/seqwise/methods.py @@ -10,6 +10,7 @@ import torch.nn as nn from vllm.config.pooler import SequencePoolingType from vllm.model_executor.layers.pooler import PoolingParamsUpdate from vllm.tasks import PoolingTask +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.pool.metadata import PoolingMetadata SequencePoolingMethodOutput: TypeAlias = torch.Tensor | list[torch.Tensor] @@ -74,15 +75,14 @@ class MeanPool(SequencePoolingMethod): # early return for empty batch return hidden_states.new_empty((0, hidden_size), dtype=torch.float32) - # Build segment_ids on CPU so repeat_interleave doesn't need to sync - # GPU->CPU to learn its data-dependent output length, then upload - # non-blocking. eg. [2, 1, 3] -> [0, 0, 1, 2, 2, 2] + prompt_lens = async_tensor_h2d( + prompt_lens_cpu, device=hidden_states.device, dtype=torch.int64 + ) + # eg. [2, 1, 3] -> [0, 0, 1, 2, 2, 2] segment_ids = torch.repeat_interleave( - torch.arange(num_seqs, dtype=torch.long), - prompt_lens_cpu, - ).to(hidden_states.device, non_blocking=True) - prompt_lens = prompt_lens_cpu.to( - hidden_states.device, dtype=torch.int64, non_blocking=True + torch.arange(num_seqs, device=hidden_states.device, dtype=torch.long), + prompt_lens, + output_size=int(prompt_lens_cpu.sum()), ) segment_sums = torch.zeros( (num_seqs, hidden_size), diff --git a/vllm/model_executor/models/moonvit.py b/vllm/model_executor/models/moonvit.py index 73e17cb9fb..56204dd3c6 100644 --- a/vllm/model_executor/models/moonvit.py +++ b/vllm/model_executor/models/moonvit.py @@ -66,6 +66,7 @@ from vllm.model_executor.models.utils import maybe_prefix from vllm.model_executor.models.vision import is_vit_use_data_parallel from vllm.platforms import current_platform from vllm.transformers_utils.configs.moonvit import MoonViTConfig +from vllm.utils.torch_utils import async_tensor_h2d def _apply_rope_input_validation(x, freqs_cis): @@ -758,7 +759,7 @@ class MoonVitPretrainedModel(PreTrainedModel): ), ] ) - metadata["cu_seqlens"] = torch.from_numpy(cu_seqlens_np).to(device) + metadata["cu_seqlens"] = async_tensor_h2d(cu_seqlens_np, device=device) if max_seqlen_override is not None: max_seqlen_val = int(max_seqlen_override) @@ -770,7 +771,7 @@ class MoonVitPretrainedModel(PreTrainedModel): metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32) gather_idx_np = _build_merge_gather_idx(grid_pairs, self.merge_kernel_size) - metadata["merge_gather_idx"] = torch.from_numpy(gather_idx_np).to(device) + metadata["merge_gather_idx"] = async_tensor_h2d(gather_idx_np, device=device) return metadata diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 04c54f1b34..986783fa34 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -83,9 +83,8 @@ from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import PromptReplacement, PromptUpdate from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from vllm.utils.platform_utils import is_pin_memory_available from vllm.utils.tensor_schema import TensorSchema, TensorShape -from vllm.utils.torch_utils import async_tensor_h2d +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers @@ -825,7 +824,7 @@ class Qwen2_5_VisionTransformer(nn.Module): @staticmethod def invert_permutation(perm: torch.Tensor) -> torch.Tensor: # building the inverse permutation in O(n) time - inv = torch.empty_like(perm, pin_memory=is_pin_memory_available()) + inv = torch.empty_like(perm, pin_memory=PIN_MEMORY) inv[perm] = torch.arange(perm.numel(), device=perm.device, dtype=perm.dtype) return inv diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index ca14fe20b1..2d2a42824c 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -13,6 +13,7 @@ from vllm.config.cache import CacheDType from vllm.platforms.interface import DeviceCapability from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv +from vllm.utils.torch_utils import np_to_pinned_tensor from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -207,7 +208,7 @@ class DeepseekV4FlashMLAMetadataBuilder( # Zero-fill for cudagraphs self.req_id_per_token_buffer.fill_(0) self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_( - torch.from_numpy(req_id_per_token), non_blocking=True + np_to_pinned_tensor(req_id_per_token), non_blocking=True ) req_id_per_token = self.req_id_per_token_buffer[:num_tokens] diff --git a/vllm/multimodal/inputs.py b/vllm/multimodal/inputs.py index d98a1624ac..c55bbe4623 100644 --- a/vllm/multimodal/inputs.py +++ b/vllm/multimodal/inputs.py @@ -488,7 +488,13 @@ class MultiModalBatchedField(BaseMultiModalField): # An optimization when `batch` contains only one tensor: # - produce exactly same result as `torch.stack(batch)` # - will achieve zero-copy if the tensor is contiguous - return batch[0].unsqueeze(0).contiguous() + out = batch[0].unsqueeze(0) + if not pin_memory: + return out.contiguous() + # Avoid extra copy - pinning unpinned memory will make it contiguous + if not out.is_contiguous() and out.is_pinned(): + out = out.contiguous() + return out.pin_memory() first_shape = batch[0].shape if all(elem.shape == first_shape for elem in batch): out = torch.empty( @@ -538,7 +544,13 @@ class MultiModalFlatField(BaseMultiModalField): # An optimization when `batch` contains only one tensor: # - produce exactly same result as `torch.concat(batch)` # - will achieve zero-copy if the tensor is contiguous - return batch[0].contiguous() + out = batch[0] + if not pin_memory: + return out.contiguous() + # Avoid extra copy - pinning unpinned memory will make it contiguous + if not out.is_contiguous() and out.is_pinned(): + out = out.contiguous() + return out.pin_memory() dim = self.dim + (self.dim < 0) * len(batch[0].shape) diff --git a/vllm/platforms/__init__.py b/vllm/platforms/__init__.py index 36692c7b76..ac536aff00 100644 --- a/vllm/platforms/__init__.py +++ b/vllm/platforms/__init__.py @@ -9,7 +9,6 @@ from typing import TYPE_CHECKING from vllm import envs from vllm.plugins import PLATFORM_PLUGINS_GROUP, load_plugins_by_group from vllm.utils.import_utils import resolve_obj_by_qualname -from vllm.utils.torch_utils import supports_xccl from .interface import CpuArchEnum, Platform, PlatformEnum @@ -135,7 +134,7 @@ def xpu_platform_plugin() -> str | None: try: import torch - if supports_xccl(): + if torch.distributed.is_xccl_available(): dist_backend = "xccl" from vllm.platforms.xpu import XPUPlatform diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index 6bf1793eef..259077da35 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -23,7 +23,6 @@ import vllm._C_stable_libtorch # noqa import vllm.envs as envs from vllm.logger import init_logger from vllm.utils.import_utils import import_pynvml -from vllm.utils.torch_utils import is_quantized_kv_cache from vllm.v1.attention.backends.registry import AttentionBackendEnum from .interface import DeviceCapability, Platform, PlatformEnum, in_wsl @@ -88,6 +87,8 @@ def _get_backend_priorities( kv_cache_dtype: CacheDType | None = None, ) -> list[AttentionBackendEnum]: """Get backend priorities with lazy import to avoid circular dependency.""" + from vllm.utils.torch_utils import is_quantized_kv_cache + if use_mla: if device_capability.major == 10: # Sparse MLA backend priorities diff --git a/vllm/platforms/xpu.py b/vllm/platforms/xpu.py index 3e208688e8..030b4933bb 100644 --- a/vllm/platforms/xpu.py +++ b/vllm/platforms/xpu.py @@ -14,7 +14,6 @@ import vllm_xpu_kernels._xpu_C # noqa import vllm.envs as envs from vllm.logger import init_logger -from vllm.utils.torch_utils import supports_xpu_graph from vllm.v1.attention.backends.registry import AttentionBackendEnum from .interface import DeviceCapability, Platform, PlatformEnum @@ -178,8 +177,6 @@ class XPUPlatform(Platform): @classmethod def check_and_update_config(cls, vllm_config: VllmConfig) -> None: - parallel_config = vllm_config.parallel_config - # lazy import to avoid circular import from vllm.config import CUDAGraphMode @@ -190,6 +187,10 @@ class XPUPlatform(Platform): attention_config = vllm_config.attention_config if attention_config.backend is None: attention_config.backend = AttentionBackendEnum.FLASH_ATTN + + # lazy import to avoid circular import + from vllm.utils.torch_utils import supports_xpu_graph + if not supports_xpu_graph(): compilation_config.cudagraph_mode = CUDAGraphMode.NONE logger.warning( @@ -324,9 +325,8 @@ class XPUPlatform(Platform): @classmethod def get_device_communicator_cls(cls) -> str: - from vllm.utils.torch_utils import supports_xccl - - if not supports_xccl(): + if not torch.distributed.is_xccl_available(): + # Supports xccl with PyTorch versions >= 2.8.0.dev for XPU platform logger.warning( "xccl is not enabled in this torch build, communication" " is not available." diff --git a/vllm/utils/torch_utils.py b/vllm/utils/torch_utils.py index 12ec5b0fcc..9269fbb44d 100644 --- a/vllm/utils/torch_utils.py +++ b/vllm/utils/torch_utils.py @@ -3,7 +3,6 @@ import contextlib import importlib.metadata import os -import platform import random import threading from collections.abc import Callable, Collection @@ -18,6 +17,7 @@ from torch.library import Library, infer_schema import vllm.envs as envs from vllm.logger import init_logger +from vllm.utils.platform_utils import is_pin_memory_available if TYPE_CHECKING: from vllm.config import ModelConfig @@ -68,9 +68,7 @@ MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP = { T = TypeVar("T") -# Pin memory in non-WSL case. -# Logic duplicated here for now to avoid circular import. -PIN_MEMORY = "microsoft" not in " ".join(platform.uname()).lower() +PIN_MEMORY = is_pin_memory_available() def is_quantized_kv_cache(kv_cache_dtype: str) -> bool: @@ -606,14 +604,24 @@ def create_kv_caches_with_random( def async_tensor_h2d( - data: list, - dtype: torch.dtype, + data: list | np.ndarray | torch.Tensor, device: str | torch.device, - pin_memory: bool = PIN_MEMORY, + dtype: torch.dtype | None = None, ) -> torch.Tensor: - """Asynchronously create a tensor and copy it from host to device.""" - t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu") - return t.to(device=device, non_blocking=True) + """Copy list/numpy array/tensor async from host to device.""" + if isinstance(data, np.ndarray): + data = torch.from_numpy(data) + if isinstance(data, torch.Tensor): + t = data.pin_memory() if PIN_MEMORY else data + else: + t = torch.tensor(data, dtype=dtype, pin_memory=PIN_MEMORY, device="cpu") + assert t.is_cpu + return t.to(device=device, dtype=dtype, non_blocking=True) + + +def np_to_pinned_tensor(array: np.ndarray) -> torch.Tensor: + t = torch.from_numpy(array) + return t.pin_memory() if PIN_MEMORY else t def make_ndarray_with_pad( @@ -914,11 +922,6 @@ def _encode_layer_name(layer_name: str) -> str | LayerName: return LayerName(layer_name) if _USE_LAYERNAME else layer_name -# Supports xccl with PyTorch versions >= 2.8.0.dev for XPU platform -def supports_xccl() -> bool: - return torch.distributed.is_xccl_available() - - # Supports XPU Graph with PyTorch versions >= 2.11.0.dev for XPU platform def supports_xpu_graph() -> bool: return is_torch_equal_or_newer("2.11.0.dev") diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 486aa7e405..666c32bca8 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -41,8 +41,8 @@ from vllm.utils.flashinfer import ( use_trtllm_attention, ) from vllm.utils.math_utils import cdiv -from vllm.utils.platform_utils import is_pin_memory_available from vllm.utils.torch_utils import ( + PIN_MEMORY, canonicalize_singleton_dim_strides, is_quantized_kv_cache, is_strictly_contiguous, @@ -708,9 +708,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # Since we do not have explicit synchronization in ModelRunnerV2, we do not pin # reused CPU buffers to avoid a race condition between step N async copies to # GPU and step N+1 buffer updates. - self.pin_memory = ( - not vllm_config.use_v2_model_runner and is_pin_memory_available() - ) + self.pin_memory = not vllm_config.use_v2_model_runner and PIN_MEMORY self.paged_kv_indptr = self._make_buffer(max_num_reqs + 1) self.paged_kv_indptr_cpu_buffer = torch.zeros_like( self.paged_kv_indptr.cpu, pin_memory=self.pin_memory diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index 829f3472dd..983544b560 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -28,7 +28,11 @@ from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv -from vllm.utils.torch_utils import is_quantized_kv_cache, is_torch_equal_or_newer +from vllm.utils.torch_utils import ( + async_tensor_h2d, + is_quantized_kv_cache, + is_torch_equal_or_newer, +) from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -58,7 +62,7 @@ def _offsets_to_doc_ids_tensor( doc_ids = torch.repeat_interleave( torch.arange(len(counts), dtype=torch.int32), counts ) - return doc_ids.to(device, non_blocking=True) + return async_tensor_h2d(doc_ids, device=device) def pad_to_multiple(x: torch.Tensor, multiple: int, dim: int): diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index 9323c5d8a4..c615ab62c1 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -8,6 +8,7 @@ from typing import Literal import torch from vllm.config import VllmConfig +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -203,8 +204,8 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] spec_sequence_masks = None spec_sequence_masks_cpu = None else: - spec_sequence_masks = spec_sequence_masks_cpu.to( - query_start_loc.device, non_blocking=True + spec_sequence_masks = async_tensor_h2d( + spec_sequence_masks_cpu, device=query_start_loc.device ) if spec_sequence_masks is None: @@ -376,12 +377,14 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] ) assert prefill_query_start_loc_cpu is not None - chunk_indices = prepare_chunk_indices( - prefill_query_start_loc_cpu, FLA_CHUNK_SIZE - ).to(device=gpu_device, non_blocking=True) - chunk_offsets = prepare_chunk_offsets( - prefill_query_start_loc_cpu, FLA_CHUNK_SIZE - ).to(device=gpu_device, non_blocking=True) + chunk_indices = async_tensor_h2d( + prepare_chunk_indices(prefill_query_start_loc_cpu, FLA_CHUNK_SIZE), + device=gpu_device, + ) + chunk_offsets = async_tensor_h2d( + prepare_chunk_offsets(prefill_query_start_loc_cpu, FLA_CHUNK_SIZE), + device=gpu_device, + ) if num_prefills > 0: has_initial_state = context_lens_tensor > 0 diff --git a/vllm/v1/attention/backends/mamba2_attn.py b/vllm/v1/attention/backends/mamba2_attn.py index 5f25c4a795..6b4999ab35 100644 --- a/vllm/v1/attention/backends/mamba2_attn.py +++ b/vllm/v1/attention/backends/mamba2_attn.py @@ -7,6 +7,7 @@ from typing import Any import torch from vllm.config import VllmConfig +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import ( AttentionBackend, CommonAttentionMetadata, @@ -68,22 +69,22 @@ def compute_varlen_chunk_metadata( # Exclusive prefix sum over logical-chunk lengths if chunk_lens: - cu_chunk_seqlens = torch.tensor( - [0] + list(itertools.accumulate(chunk_lens)), - device=device, - dtype=torch.int32, - ) - # Final boundary must equal total tokens - assert int(cu_chunk_seqlens[-1].item()) == total + cu_chunk_seqlens_list = [0] + list(itertools.accumulate(chunk_lens)) + # Final boundary must equal total tokens (check on host to avoid a sync) + assert cu_chunk_seqlens_list[-1] == total else: - cu_chunk_seqlens = torch.tensor([0], device=device, dtype=torch.int32) + cu_chunk_seqlens_list = [0] + cu_chunk_seqlens = async_tensor_h2d( + cu_chunk_seqlens_list, dtype=torch.int32, device=device + ) - last_chunk_indices_t = ( - torch.tensor(last_chunk_indices, device=device, dtype=torch.int32) - if len(starts) > 0 - else torch.empty((0,), device=device, dtype=torch.int32) + # last_chunk_indices is empty when there are no sequences (len(starts) == 0). + last_chunk_indices_t = async_tensor_h2d( + last_chunk_indices, dtype=torch.int32, device=device + ) + seq_idx_chunks_t = async_tensor_h2d( + seq_idx_chunks, dtype=torch.int32, device=device ) - seq_idx_chunks_t = torch.tensor(seq_idx_chunks, device=device, dtype=torch.int32) return cu_chunk_seqlens, last_chunk_indices_t, seq_idx_chunks_t diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index 01716f567d..5547c62649 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -26,7 +26,7 @@ from vllm.model_executor.layers.attention.mla_attention import ( get_mla_dims, ) from vllm.platforms.interface import DeviceCapability -from vllm.utils.torch_utils import is_quantized_kv_cache +from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -217,7 +217,7 @@ class FlashInferMLASparseMetadataBuilder( # Zero-fill for cudagraphs self.req_id_per_token_buffer.fill_(0) self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_( - torch.from_numpy(req_id_per_token), non_blocking=True + np_to_pinned_tensor(req_id_per_token), non_blocking=True ) req_id_per_token_tensor = self.req_id_per_token_buffer[:num_tokens] diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 6d8dfe1312..19381efd73 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -16,7 +16,7 @@ from vllm.model_executor.layers.attention.mla_attention import ( from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.utils.platform_utils import num_compute_units -from vllm.utils.torch_utils import is_quantized_kv_cache +from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -503,7 +503,7 @@ class FlashMLASparseMetadataBuilder(AttentionMetadataBuilder[FlashMLASparseMetad # Zero-fill for cudagraphs self.req_id_per_token_buffer.fill_(0) self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_( - torch.from_numpy(req_id_per_token), non_blocking=True + np_to_pinned_tensor(req_id_per_token), non_blocking=True ) req_id_per_token = self.req_id_per_token_buffer[:num_tokens] diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index 30db5d5f5a..1b7b8a01a5 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -17,7 +17,7 @@ from typing_extensions import runtime_checkable from vllm.config import VllmConfig, get_layers_from_vllm_config from vllm.utils.math_utils import cdiv -from vllm.utils.torch_utils import async_tensor_h2d +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec if TYPE_CHECKING: @@ -364,8 +364,8 @@ def make_local_attention_virtual_batches( # tensor first, which recovers perf. # Upload the index tensors to the block_table's device up-front so that the # fancy indexing below doesn't implicitly force a synchronous H2D copy. - batch_indices_torch = torch.from_numpy(batch_indices).to(device, non_blocking=True) - block_indices_torch = torch.from_numpy(block_indices).to(device, non_blocking=True) + batch_indices_torch = async_tensor_h2d(batch_indices, device=device) + block_indices_torch = async_tensor_h2d(block_indices, device=device) # Save as a lambda so we can return this for update_block_table make_block_table = lambda block_table: block_table[ @@ -379,8 +379,8 @@ def make_local_attention_virtual_batches( return CommonAttentionMetadata( query_start_loc_cpu=query_start_loc_cpu, - query_start_loc=query_start_loc_cpu.to(device=device, non_blocking=True), - seq_lens=seq_lens_cpu.to(device=device, non_blocking=True), + query_start_loc=async_tensor_h2d(query_start_loc_cpu, device=device), + seq_lens=async_tensor_h2d(seq_lens_cpu, device=device), num_reqs=len(seq_lens_cpu), num_actual_tokens=common_attn_metadata.num_actual_tokens, max_query_len=seqlens_q_local.max(), @@ -808,14 +808,12 @@ def create_fast_prefill_custom_backend( def compute_causal_conv1d_metadata( - query_start_loc_p_cpu: torch.Tensor, - *, - device: torch.device, -): + query_start_loc_p_cpu: torch.Tensor, *, device: torch.device +) -> tuple[dict[int, dict[str, Any]], torch.Tensor, torch.Tensor]: # Needed for causal_conv1d. Use the CPU query_start_loc to avoid DtoH sync. assert query_start_loc_p_cpu.device.type == "cpu" seqlens = query_start_loc_p_cpu.diff() - nums_dict = {} # type: ignore + nums_dict: dict[int, dict[str, Any]] = {} batch_ptr = None token_chunk_offset_ptr = None for BLOCK_M in [8]: # cover all BLOCK_M values @@ -823,7 +821,7 @@ def compute_causal_conv1d_metadata( nums_dict[BLOCK_M] = {} nums_dict[BLOCK_M]["nums"] = nums nums_dict[BLOCK_M]["tot"] = nums.sum().item() - mlist = torch.from_numpy(np.repeat(np.arange(len(nums)), nums)) + mlist = np_to_pinned_tensor(np.repeat(np.arange(len(nums)), nums)) nums_dict[BLOCK_M]["mlist"] = mlist mlist_len = len(nums_dict[BLOCK_M]["mlist"]) nums_dict[BLOCK_M]["mlist_len"] = mlist_len @@ -831,7 +829,7 @@ def compute_causal_conv1d_metadata( offsetlist = [] # type: ignore for idx, num in enumerate(nums): offsetlist.extend(range(num)) - offsetlist = torch.tensor(offsetlist, dtype=torch.int32) + offsetlist = torch.tensor(offsetlist, dtype=torch.int32, pin_memory=PIN_MEMORY) nums_dict[BLOCK_M]["offsetlist"] = offsetlist if batch_ptr is None: @@ -845,16 +843,15 @@ def compute_causal_conv1d_metadata( else: if batch_ptr.nelement() < MAX_NUM_PROGRAMS: batch_ptr.resize_(MAX_NUM_PROGRAMS).fill_(PAD_SLOT_ID) - token_chunk_offset_ptr.resize_( # type: ignore - MAX_NUM_PROGRAMS - ).fill_(PAD_SLOT_ID) + assert token_chunk_offset_ptr is not None + token_chunk_offset_ptr.resize_(MAX_NUM_PROGRAMS).fill_(PAD_SLOT_ID) + assert batch_ptr is not None batch_ptr[0:mlist_len].copy_(mlist, non_blocking=True) - token_chunk_offset_ptr[ # type: ignore - 0:mlist_len - ].copy_(offsetlist, non_blocking=True) + assert token_chunk_offset_ptr is not None + token_chunk_offset_ptr[0:mlist_len].copy_(offsetlist, non_blocking=True) nums_dict[BLOCK_M]["batch_ptr"] = batch_ptr - nums_dict[BLOCK_M]["token_chunk_offset_ptr"] = token_chunk_offset_ptr # type: ignore + nums_dict[BLOCK_M]["token_chunk_offset_ptr"] = token_chunk_offset_ptr return nums_dict, batch_ptr, token_chunk_offset_ptr diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index 81545281b6..f4d3869dc1 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -14,7 +14,7 @@ from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON, triton from vllm.utils.math_utils import cdiv -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.kv_offload.base import ( BlockIDsLoadStoreSpec, CanonicalKVCacheRef, @@ -156,7 +156,7 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None: def _new_descriptor_buffers( num_copy_ops: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - pin = is_pin_memory_available() + pin = PIN_MEMORY # CUDA cache_kernels.cu requires int64; XPU DMA engine requires uint64. ptr_dtype = torch.uint64 if current_platform.is_xpu() else torch.int64 return ( @@ -482,7 +482,7 @@ class CpuGpuOffloadingHandlers: num_cpu_blocks: int, mmap_region: SharedOffloadRegion | None = None, ): - pin_memory = is_pin_memory_available() + pin_memory = PIN_MEMORY logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors)) self._mmap_region = mmap_region if mmap_region is not None and pin_memory: diff --git a/vllm/v1/pool/metadata.py b/vllm/v1/pool/metadata.py index f772c850f0..9a9bb2b0e7 100644 --- a/vllm/v1/pool/metadata.py +++ b/vllm/v1/pool/metadata.py @@ -7,9 +7,7 @@ import torch from vllm.pooling_params import PoolingParams from vllm.tasks import PoolingTask -from vllm.utils.platform_utils import is_pin_memory_available - -pin_memory = is_pin_memory_available() +from vllm.utils.torch_utils import PIN_MEMORY @dataclass @@ -134,7 +132,7 @@ class PoolingMetadata: num_scheduled_tokens_cpu = torch.from_numpy(num_scheduled_tokens_np) if query_start_loc_gpu is None: cumsum = torch.zeros( - n_seq + 1, dtype=torch.int64, pin_memory=pin_memory, device="cpu" + n_seq + 1, dtype=torch.int64, pin_memory=PIN_MEMORY, device="cpu" ) torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:]) cumsum = cumsum.to(device, non_blocking=True) diff --git a/vllm/v1/sample/logits_processor/builtin.py b/vllm/v1/sample/logits_processor/builtin.py index 11a52711d6..d7c9444380 100644 --- a/vllm/v1/sample/logits_processor/builtin.py +++ b/vllm/v1/sample/logits_processor/builtin.py @@ -7,6 +7,7 @@ import numpy as np import torch from vllm import SamplingParams +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.sample.logits_processor.interface import ( BatchUpdate, LogitsProcessor, @@ -118,7 +119,6 @@ class MinPLogitsProcessor(LogitsProcessor): class LogitBiasLogitsProcessor(LogitsProcessor): def __init__(self, _, device: torch.device, is_pin_memory: bool): self.device = device - self.pin_memory = is_pin_memory self.biases: dict[int, dict[int, float]] = {} self.bias_tensor: torch.Tensor = torch.tensor(()) @@ -154,9 +154,7 @@ class LogitBiasLogitsProcessor(LogitsProcessor): ) def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor: - return torch.tensor( - data, device="cpu", dtype=dtype, pin_memory=self.pin_memory - ).to(device=self.device, non_blocking=True) + return async_tensor_h2d(data, device=self.device, dtype=dtype) def apply(self, logits: torch.Tensor) -> torch.Tensor: if self.biases: @@ -170,7 +168,6 @@ class MinTokensLogitsProcessor(LogitsProcessor): ): # index -> (min_toks, output_token_ids, stop_token_ids) self.device = device - self.pin_memory = is_pin_memory self.min_toks: dict[int, tuple[int, Sequence[int], set[int]]] = {} # (req_idx_tensor,eos_tok_id_tensor) @@ -227,9 +224,7 @@ class MinTokensLogitsProcessor(LogitsProcessor): ) def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor: - return torch.tensor( - data, device="cpu", dtype=dtype, pin_memory=self.pin_memory - ).to(device=self.device, non_blocking=True) + return async_tensor_h2d(data, device=self.device, dtype=dtype) def apply(self, logits: torch.Tensor) -> torch.Tensor: if self.min_toks: @@ -283,8 +278,8 @@ class MinTokensLogitsProcessor(LogitsProcessor): toks_arr = np.concatenate(all_toks) # (row_indices, token_indices) for index_put_ to set -inf. logits_slice = ( - torch.from_numpy(rows_arr).to(self.device, non_blocking=True), - torch.from_numpy(toks_arr).to(self.device, non_blocking=True), + async_tensor_h2d(rows_arr, device=self.device), + async_tensor_h2d(toks_arr, device=self.device), ) logits.index_put_(logits_slice, self.neg_inf_tensor) diff --git a/vllm/v1/sample/ops/penalties.py b/vllm/v1/sample/ops/penalties.py index 241d9de957..7bc6ec7ab8 100644 --- a/vllm/v1/sample/ops/penalties.py +++ b/vllm/v1/sample/ops/penalties.py @@ -4,8 +4,7 @@ import torch from vllm.model_executor.layers.utils import apply_penalties -from vllm.utils.platform_utils import is_pin_memory_available -from vllm.utils.torch_utils import make_tensor_with_pad +from vllm.utils.torch_utils import PIN_MEMORY, make_tensor_with_pad def apply_all_penalties( @@ -52,6 +51,6 @@ def _convert_to_tensors( pad=vocab_size, device="cpu", dtype=torch.int64, - pin_memory=is_pin_memory_available(), + pin_memory=PIN_MEMORY, ) return output_tokens_tensor.to(device, non_blocking=True) diff --git a/vllm/v1/sample/sampler.py b/vllm/v1/sample/sampler.py index eadc009c25..bb20432a08 100644 --- a/vllm/v1/sample/sampler.py +++ b/vllm/v1/sample/sampler.py @@ -6,7 +6,7 @@ import torch import torch.nn as nn from vllm.config.model import LogprobsMode -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.outputs import LogprobsTensors, SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata from vllm.v1.sample.ops.bad_words import apply_bad_words @@ -65,7 +65,7 @@ class Sampler(nn.Module): ): super().__init__() self.topk_topp_sampler = TopKTopPSampler(logprobs_mode, use_fp64_gumbel) - self.pin_memory = is_pin_memory_available() + self.pin_memory = PIN_MEMORY self.logprobs_mode = logprobs_mode self.use_fp64_gumbel = use_fp64_gumbel diff --git a/vllm/v1/sample/thinking_budget_state.py b/vllm/v1/sample/thinking_budget_state.py index 8789e6afdc..d32d1b3029 100644 --- a/vllm/v1/sample/thinking_budget_state.py +++ b/vllm/v1/sample/thinking_budget_state.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any import torch from vllm.platforms import current_platform -from vllm.utils.torch_utils import async_tensor_h2d +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from vllm.v1.sample.logits_processor.interface import ( BatchUpdate, MoveDirectionality, @@ -22,12 +22,11 @@ def maybe_create_thinking_budget_state_holder( max_num_seqs: int, num_spec_tokens: int, device: torch.device, - is_pin_memory: bool, ) -> "ThinkingBudgetStateHolder | None": if reasoning_config is None: return None return ThinkingBudgetStateHolder( - reasoning_config, max_num_seqs, num_spec_tokens, device, is_pin_memory + reasoning_config, max_num_seqs, num_spec_tokens, device, PIN_MEMORY ) diff --git a/vllm/v1/serial_utils.py b/vllm/v1/serial_utils.py index 204c8bd0e4..bc4619a7eb 100644 --- a/vllm/v1/serial_utils.py +++ b/vllm/v1/serial_utils.py @@ -33,7 +33,7 @@ from vllm.multimodal.inputs import ( MultiModalSharedField, NestedTensors, ) -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.utils import tensor_data logger = init_logger(__name__) @@ -327,7 +327,7 @@ class MsgpackDecoder: oob_tensor_provider: OOBTensorProvider | None = None, ): self.share_mem = share_mem - self.pin_tensors = is_pin_memory_available() + self.pin_tensors = PIN_MEMORY args = () if t is None else (t,) self.decoder = msgpack.Decoder( *args, ext_hook=self.ext_hook, dec_hook=self.dec_hook diff --git a/vllm/v1/simple_kv_offload/worker.py b/vllm/v1/simple_kv_offload/worker.py index c23b44f291..d33e5f7620 100644 --- a/vllm/v1/simple_kv_offload/worker.py +++ b/vllm/v1/simple_kv_offload/worker.py @@ -8,7 +8,7 @@ import torch from vllm.config import VllmConfig from vllm.logger import init_logger -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.simple_kv_offload.copy_backend import DmaCopyBackend from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor from vllm.v1.simple_kv_offload.metadata import ( @@ -149,7 +149,7 @@ class SimpleCPUOffloadWorker: (self.num_cpu_blocks * total_bytes_per_block) / (1024**3), ) - pin_memory = is_pin_memory_available() + pin_memory = PIN_MEMORY if not pin_memory: logger.warning( "Pinned memory not available. CPU offload performance may be degraded." diff --git a/vllm/v1/spec_decode/extract_hidden_states.py b/vllm/v1/spec_decode/extract_hidden_states.py index a0a1f03c71..b6f9eac4df 100644 --- a/vllm/v1/spec_decode/extract_hidden_states.py +++ b/vllm/v1/spec_decode/extract_hidden_states.py @@ -12,7 +12,7 @@ from vllm.config import CUDAGraphMode, VllmConfig, get_layers_from_vllm_config from vllm.forward_context import set_forward_context from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.model_loader import get_model -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.attention.backend import AttentionMetadataBuilder, CommonAttentionMetadata from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher from vllm.v1.utils import CpuGpuBuffer @@ -58,7 +58,7 @@ class ExtractHiddenStatesProposer: self.backup_next_token_ids = CpuGpuBuffer( max_batch_size, dtype=torch.int32, - pin_memory=is_pin_memory_available(), + pin_memory=PIN_MEMORY, device=device, with_numpy=True, ) @@ -317,7 +317,6 @@ class ExtractHiddenStatesProposer: (batch_size, 1). For each request we either use the sampled token (if valid and not discarded) or a backup token from the request state. """ - num_reqs = gpu_input_batch.num_reqs # Precompute backup token IDs for discarded requests. num_reqs = gpu_input_batch.num_reqs diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index b7c01d3ec1..bdc10313f4 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -26,7 +26,7 @@ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.platforms import current_platform -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadata @@ -228,7 +228,7 @@ class SpecDecodeBaseProposer: self.backup_next_token_ids = CpuGpuBuffer( self.max_batch_size, dtype=torch.int32, - pin_memory=is_pin_memory_available(), + pin_memory=PIN_MEMORY, device=device, with_numpy=True, ) @@ -239,9 +239,7 @@ class SpecDecodeBaseProposer: self._last_draft_probs: torch.Tensor | None = None self._slot_mapping_buffer = torch.zeros( - self.max_positions, - dtype=torch.int64, - device=device, + self.max_positions, dtype=torch.int64, device=device ) # Determine allowed attention backends once during initialization. @@ -1127,7 +1125,7 @@ class SpecDecodeBaseProposer: new_query_start_loc_cpu = torch.zeros( query_start_loc_cpu.shape, dtype=torch.int32, - pin_memory=is_pin_memory_available(), + pin_memory=PIN_MEMORY, ) new_query_start_loc_np = new_query_start_loc_cpu.numpy() np.cumsum(new_num_tokens_per_req_np, out=new_query_start_loc_np[1:]) @@ -1160,11 +1158,11 @@ class SpecDecodeBaseProposer: # q1 + 0, q1 + 1, q1 + 2, q1 + 3, // req 2 # q1 + q2 + 0, q1 + q2 + 1, q1 + q2 + 2] // req 3 token_indices_np = token_offsets + old_query_start_locs_expanded - token_indices = torch.from_numpy(token_indices_np).to(device, non_blocking=True) + token_indices = async_tensor_h2d(token_indices_np, device=device) spec_common_attn_metadata = CommonAttentionMetadata( - query_start_loc=new_query_start_loc_cpu.to(device, non_blocking=True), - seq_lens=new_seq_lens_cpu.to(device, non_blocking=True), + query_start_loc=async_tensor_h2d(new_query_start_loc_cpu, device=device), + seq_lens=async_tensor_h2d(new_seq_lens_cpu, device=device), query_start_loc_cpu=new_query_start_loc_cpu, _seq_lens_cpu=new_seq_lens_cpu, _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, diff --git a/vllm/v1/spec_decode/ngram_proposer_gpu.py b/vllm/v1/spec_decode/ngram_proposer_gpu.py index b8a0116ede..ed544bb27c 100644 --- a/vllm/v1/spec_decode/ngram_proposer_gpu.py +++ b/vllm/v1/spec_decode/ngram_proposer_gpu.py @@ -545,7 +545,7 @@ def update_ngram_gpu_tensors_incremental( num_tokens = input_batch.num_tokens_no_spec[idx] if num_tokens > 0: token_ids_gpu_tensor[idx, :num_tokens].copy_( - input_batch.token_ids_cpu_tensor[idx, :num_tokens], + input_batch.token_ids_cpu_tensor[idx, :num_tokens].pin_memory(), non_blocking=True, ) @@ -591,7 +591,7 @@ def update_ngram_gpu_tensors_incremental( num_tokens = input_batch.num_tokens_no_spec[new_req_idx] if num_tokens > 0: token_ids_gpu_tensor[new_req_idx, :num_tokens].copy_( - input_batch.token_ids_cpu_tensor[new_req_idx, :num_tokens], + input_batch.token_ids_cpu_tensor[new_req_idx, :num_tokens].pin_memory(), non_blocking=True, ) diff --git a/vllm/v1/structured_output/backend_lm_format_enforcer.py b/vllm/v1/structured_output/backend_lm_format_enforcer.py index 94568b09a7..bbda96e60b 100644 --- a/vllm/v1/structured_output/backend_lm_format_enforcer.py +++ b/vllm/v1/structured_output/backend_lm_format_enforcer.py @@ -11,7 +11,7 @@ from transformers import PreTrainedTokenizerBase from vllm.sampling_params import SamplingParams from vllm.utils.import_utils import LazyLoader -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.structured_output.backend_types import ( StructuredOutputBackend, StructuredOutputGrammar, @@ -139,7 +139,7 @@ class LMFormatEnforcerBackend(StructuredOutputBackend): (max_num_seqs, (self.vocab_size + 31) // 32), -1, dtype=torch.int32, - pin_memory=is_pin_memory_available(), + pin_memory=PIN_MEMORY, ) def destroy(self): diff --git a/vllm/v1/structured_output/backend_outlines.py b/vllm/v1/structured_output/backend_outlines.py index 71dd5d8064..91627ff154 100644 --- a/vllm/v1/structured_output/backend_outlines.py +++ b/vllm/v1/structured_output/backend_outlines.py @@ -15,7 +15,7 @@ from regex import escape as regex_escape from vllm.sampling_params import SamplingParams from vllm.utils.import_utils import LazyLoader -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.structured_output.backend_types import ( StructuredOutputBackend, StructuredOutputGrammar, @@ -101,7 +101,7 @@ class OutlinesBackend(StructuredOutputBackend): (max_num_seqs, (self.vocab_size + 31) // 32), -1, dtype=torch.int32, - pin_memory=is_pin_memory_available(), + pin_memory=PIN_MEMORY, ) def destroy(self): diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index d30dcf2617..cde31e0fd5 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -10,7 +10,6 @@ from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, TimeoutError from typing import TYPE_CHECKING, TypeVar -import numpy as np import regex as re import torch from cachetools import LRUCache @@ -18,7 +17,7 @@ from cachetools import LRUCache import vllm.envs as envs from vllm.logger import init_logger from vllm.utils.import_utils import LazyLoader -from vllm.utils.platform_utils import is_pin_memory_available +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput if TYPE_CHECKING: @@ -123,11 +122,13 @@ def apply_grammar_bitmask( out_indices = [] # Reorder the bitmask to match the order of the requests in the batch. - sorted_bitmask = np.full( - shape=(logits.shape[0], grammar_bitmask.shape[1]), - fill_value=-1, - dtype=grammar_bitmask.dtype, + sorted_bitmask_tensor = torch.full( + (logits.shape[0], grammar_bitmask.shape[1]), + -1, + dtype=torch.from_numpy(grammar_bitmask[:0]).dtype, + pin_memory=PIN_MEMORY, ) + sorted_bitmask = sorted_bitmask_tensor.numpy() cumulative_index = 0 for req_id in grammar_output.structured_output_request_ids: num_spec_tokens = len(spec_tokens.get(req_id, ())) @@ -138,10 +139,8 @@ def apply_grammar_bitmask( out_indices.append(bitmask_index) cumulative_index += 1 + num_spec_tokens - # Copy async to device as tensor. - grammar_bitmask = torch.from_numpy(sorted_bitmask).to( - logits.device, non_blocking=True - ) + # Copy async to device. + grammar_bitmask = sorted_bitmask_tensor.to(logits.device, non_blocking=True) # If the length of out indices and the logits have the same shape # we don't need to pass indices to the kernel, @@ -154,11 +153,9 @@ def apply_grammar_bitmask( # xgrammar expects a python list of indices but it will actually work with # a tensor. If we copy the tensor ourselves here we can do it in a # non_blocking manner and there should be no cpu sync within xgrammar. - pin_memory = is_pin_memory_available() - index_tensor = torch.tensor( - out_indices, dtype=torch.int32, device="cpu", pin_memory=pin_memory + index_tensor = async_tensor_h2d( + out_indices, dtype=torch.int32, device=logits.device ) - index_tensor = index_tensor.to(logits.device, non_blocking=True) xgr.apply_token_bitmask_inplace(logits, grammar_bitmask, indices=index_tensor) return diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index ba66358c66..71ade9c860 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -31,6 +31,7 @@ from vllm.logger import init_logger from vllm.usage.usage_lib import UsageContext, is_usage_stats_enabled, usage_message from vllm.utils.network_utils import get_open_zmq_ipc_path, get_tcp_uri from vllm.utils.system_utils import decorate_logs, kill_process_tree, set_process_title +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.core.sched.output import SchedulerOutput if TYPE_CHECKING: @@ -114,7 +115,7 @@ class CpuGpuBuffer: *size: int | torch.SymInt, dtype: torch.dtype, device: torch.device, - pin_memory: bool, + pin_memory: bool = PIN_MEMORY, with_numpy: bool = True, ) -> None: # these buffers are mutable runtime state, so allocate them as normal diff --git a/vllm/v1/worker/cpu/shm.py b/vllm/v1/worker/cpu/shm.py index 92aa1b5b95..9399b823ce 100644 --- a/vllm/v1/worker/cpu/shm.py +++ b/vllm/v1/worker/cpu/shm.py @@ -7,6 +7,8 @@ from typing import Any +import numpy as np + # Patch torch APIs import torch @@ -45,11 +47,14 @@ import vllm.utils.torch_utils as torch_utils def async_tensor_h2d( - data: list, - dtype: torch.dtype, + data: list | np.ndarray | torch.Tensor, device: str | torch.device, - pin_memory: bool = False, + dtype: torch.dtype | None = None, ) -> torch.Tensor: + if isinstance(data, np.ndarray): + data = torch.from_numpy(data) + if isinstance(data, torch.Tensor): + return data.to(dtype=dtype) return torch.tensor(data, dtype=dtype, device="cpu") diff --git a/vllm/v1/worker/gpu/buffer_utils.py b/vllm/v1/worker/gpu/buffer_utils.py index cf5b2c1a2d..f8336fa074 100644 --- a/vllm/v1/worker/gpu/buffer_utils.py +++ b/vllm/v1/worker/gpu/buffer_utils.py @@ -36,10 +36,9 @@ def async_copy_to_gpu( assert device is not None out = torch.empty_like(x, device=device) - # Copy directly to GPU — explicit pin_memory() causes sporadic stalls - # under high concurrency due to CUDA driver contention. The driver - # handles the transfer efficiently without manual pinning. - return out.copy_(x, non_blocking=True) + # pin_memory() is no-op if the memory is already pinned. + pinned = x.pin_memory() + return out.copy_(pinned, non_blocking=True) class UvaBuffer: @@ -183,7 +182,7 @@ class StagedWriteTensor: # Special handling for write_contents write_contents = async_tensor_h2d( - self._staged_write_contents, self.dtype, self.device + self._staged_write_contents, device=self.device, dtype=self.dtype ) # Write diffs to the GPU buffer @@ -255,7 +254,7 @@ class FusedStagedWriter: indices_uva = self.indices.copy_to_uva(indices) starts_uva = self.starts.copy_to_uva(starts) cu_lens_uva = self.cu_lens.copy_to_uva(cu_lens) - contents_gpu = async_tensor_h2d(contents, torch.int32, self.device) + contents_gpu = async_tensor_h2d(contents, device=self.device, dtype=torch.int32) _apply_write_kernel[(len(group_ids),)]( output_ptrs, diff --git a/vllm/v1/worker/gpu/mm/encoder_runner.py b/vllm/v1/worker/gpu/mm/encoder_runner.py index 7c813c9b84..d86a166fbb 100644 --- a/vllm/v1/worker/gpu/mm/encoder_runner.py +++ b/vllm/v1/worker/gpu/mm/encoder_runner.py @@ -49,12 +49,11 @@ class EncoderRunner: @torch.inference_mode() def execute_mm_encoder( - self, - mm_kwargs: list[tuple[str, MultiModalKwargsItem]], + self, mm_kwargs: list[tuple[str, MultiModalKwargsItem]] ) -> list[torch.Tensor]: encoder_outputs: list[torch.Tensor] = [] for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs( - mm_kwargs, device=self.device, pin_memory=False + mm_kwargs, device=self.device, pin_memory=True ): batch_outputs = self.model.embed_multimodal(**mm_kwargs_batch) sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index a96068dd91..124eb10186 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -46,8 +46,7 @@ from vllm.sequence import IntermediateTensors from vllm.tasks import SupportedTask from vllm.utils.math_utils import cdiv from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib -from vllm.utils.platform_utils import is_pin_memory_available -from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE +from vllm.utils.torch_utils import PIN_MEMORY, STR_DTYPE_TO_TORCH_DTYPE from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput @@ -498,7 +497,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): """Build KV-block zeroing metadata; invoked from gpu_worker.""" self.kv_block_zeroer = KVBlockZeroer( self.device, - is_pin_memory_available(), + pin_memory=PIN_MEMORY, attn_groups_iter=(g for groups in self.attn_groups for g in groups), kernel_block_sizes=self.kernel_block_sizes, cache_dtype=self.cache_config.cache_dtype, diff --git a/vllm/v1/worker/gpu/model_states/whisper.py b/vllm/v1/worker/gpu/model_states/whisper.py index fc2909de03..df432d82ce 100644 --- a/vllm/v1/worker/gpu/model_states/whisper.py +++ b/vllm/v1/worker/gpu/model_states/whisper.py @@ -170,7 +170,8 @@ class WhisperModelState(ModelState): for_capture: bool, num_reqs: int, ) -> dict[int, tuple[torch.Tensor, np.ndarray]]: - encoder_seq_lens_np = np.zeros(num_reqs, dtype=np.int32) + encoder_seq_lens = torch.zeros(num_reqs, dtype=torch.int32, pin_memory=True) + encoder_seq_lens_np = encoder_seq_lens.numpy() if not for_capture: # During normal execution, use actual encoder lengths. for i, req_id in enumerate(req_ids): @@ -183,9 +184,7 @@ class WhisperModelState(ModelState): # is captured with the correct value for cross-attention. encoder_seq_lens_np[:] = self.max_encoder_len - self.encoder_seq_lens_gpu[:num_reqs].copy_( - torch.from_numpy(encoder_seq_lens_np), non_blocking=True - ) + self.encoder_seq_lens_gpu[:num_reqs].copy_(encoder_seq_lens, non_blocking=True) self.encoder_seq_lens_gpu[num_reqs:].fill_(0) encoder_seq_lens_gpu = self.encoder_seq_lens_gpu[:num_reqs] diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 89d69c0bde..28d1a04b78 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -15,6 +15,7 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams, SamplingType from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.utils.collection_utils import swap_dict_values +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.outputs import LogprobsTensors from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates from vllm.v1.sample.logits_processor import ( @@ -95,7 +96,6 @@ class InputBatch: max_model_len: int, max_num_batched_tokens: int, device: torch.device, - pin_memory: bool, vocab_size: int, block_sizes: list[int], # The block_size of each kv cache group kernel_block_sizes: list[int], @@ -112,7 +112,6 @@ class InputBatch: max_num_reqs, num_spec_tokens, device, - pin_memory, ) self.thinking_token_budget_reqs: set[str] = set() self.is_pooling_model = is_pooling_model @@ -120,7 +119,6 @@ class InputBatch: self.max_model_len = max_model_len self.max_num_batched_tokens = max_num_batched_tokens self.device = device - self.pin_memory = pin_memory self.vocab_size = vocab_size self._req_ids: list[str | None] = [] @@ -138,7 +136,10 @@ class InputBatch: ) self.token_ids_cpu = self.token_ids_cpu_tensor.numpy() self.is_token_ids_tensor = torch.zeros( - (max_num_reqs, max_model_len), device="cpu", dtype=bool, pin_memory=False + (max_num_reqs, max_model_len), + device="cpu", + dtype=bool, + pin_memory=False, ) self.is_token_ids = self.is_token_ids_tensor.numpy() # Store prompt embeddings per request to avoid OOM from large upfront @@ -149,21 +150,21 @@ class InputBatch: (max_num_reqs,), device="cpu", dtype=torch.int32, - pin_memory=pin_memory, + pin_memory=PIN_MEMORY, ) self.num_tokens_no_spec = self.num_tokens_no_spec_cpu_tensor.numpy() self.num_prompt_tokens_cpu_tensor = torch.zeros( (max_num_reqs,), device="cpu", dtype=torch.int32, - pin_memory=pin_memory, + pin_memory=PIN_MEMORY, ) self.num_prompt_tokens = self.num_prompt_tokens_cpu_tensor.numpy() self.num_computed_tokens_cpu_tensor = torch.zeros( (max_num_reqs,), device="cpu", dtype=torch.int32, - pin_memory=pin_memory, + pin_memory=PIN_MEMORY, ) self.num_computed_tokens_cpu = self.num_computed_tokens_cpu_tensor.numpy() @@ -172,7 +173,7 @@ class InputBatch: max_num_reqs=max_num_reqs, max_model_len=max_model_len, max_num_batched_tokens=max_num_batched_tokens, - pin_memory=pin_memory, + pin_memory=PIN_MEMORY, device=device, block_sizes=block_sizes, kernel_block_sizes=kernel_block_sizes, @@ -185,7 +186,7 @@ class InputBatch: (max_num_reqs,), dtype=torch.float32, device=device ) self.temperature_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory + (max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=PIN_MEMORY ) self.temperature_cpu = self.temperature_cpu_tensor.numpy() self.greedy_reqs: set[str] = set() @@ -193,14 +194,14 @@ class InputBatch: self.top_p = torch.empty((max_num_reqs,), dtype=torch.float32, device=device) self.top_p_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory + (max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=PIN_MEMORY ) self.top_p_cpu = self.top_p_cpu_tensor.numpy() self.top_p_reqs: set[str] = set() self.top_k = torch.empty((max_num_reqs,), dtype=torch.int32, device=device) self.top_k_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=pin_memory + (max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY ) self.top_k_cpu = self.top_k_cpu_tensor.numpy() self.top_k_reqs: set[str] = set() @@ -210,7 +211,7 @@ class InputBatch: (max_num_reqs,), dtype=torch.float, device=device ) self.frequency_penalties_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory + (max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=PIN_MEMORY ) self.frequency_penalties_cpu = self.frequency_penalties_cpu_tensor.numpy() self.frequency_penalties_reqs: set[str] = set() @@ -220,7 +221,7 @@ class InputBatch: (max_num_reqs,), dtype=torch.float, device=device ) self.presence_penalties_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory + (max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=PIN_MEMORY ) self.presence_penalties_cpu = self.presence_penalties_cpu_tensor.numpy() self.presence_penalties_reqs: set[str] = set() @@ -230,14 +231,14 @@ class InputBatch: (max_num_reqs,), dtype=torch.float, device=device ) self.repetition_penalties_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory + (max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=PIN_MEMORY ) self.repetition_penalties_cpu = self.repetition_penalties_cpu_tensor.numpy() self.repetition_penalties_reqs: set[str] = set() # Speculative decoding self.num_accepted_tokens_cpu_tensor = torch.ones( - (max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=pin_memory + (max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY ) self.num_accepted_tokens_cpu = self.num_accepted_tokens_cpu_tensor.numpy() @@ -963,7 +964,7 @@ class InputBatch: (self.num_reqs, max_prompt_len), device="cpu", dtype=torch.int64, - pin_memory=self.pin_memory, + pin_memory=PIN_MEMORY, ) prompt_token_ids = prompt_token_ids_cpu_tensor.numpy() prompt_token_ids[:] = self.token_ids_cpu[:num_reqs, :max_prompt_len] diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 3221dc46c6..b554542e65 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -116,8 +116,10 @@ from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.utils.math_utils import cdiv, round_up from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib from vllm.utils.nvtx_pytorch_hooks import PytHooks -from vllm.utils.platform_utils import is_pin_memory_available, num_compute_units +from vllm.utils.platform_utils import num_compute_units from vllm.utils.torch_utils import ( + PIN_MEMORY, + async_tensor_h2d, get_dtype_size, is_quantized_kv_cache, kv_cache_dtype_str_to_dtype, @@ -441,7 +443,6 @@ class GPUModelRunner( scheduler_config = self.scheduler_config parallel_config = self.parallel_config self.device = device - self.pin_memory = is_pin_memory_available() self.dtype = self.model_config.dtype self.kv_cache_dtype = kv_cache_dtype_str_to_dtype( @@ -666,7 +667,6 @@ class GPUModelRunner( max_model_len=max(self.max_model_len, self.max_encoder_len), max_num_batched_tokens=self.max_num_tokens, device=self.device, - pin_memory=self.pin_memory, vocab_size=self.model_config.get_vocab_size(), block_sizes=[placeholder_block_size], kernel_block_sizes=[placeholder_block_size], @@ -674,7 +674,7 @@ class GPUModelRunner( logitsprocs=build_logitsprocs( self.vllm_config, self.device, - self.pin_memory, + PIN_MEMORY, self.is_pooling_model, custom_logitsprocs, ), @@ -729,7 +729,7 @@ class GPUModelRunner( self.max_num_reqs, dtype=torch.int32, device=self.device ) self.optimistic_seq_lens_cpu = torch.zeros( - self.max_num_reqs, dtype=torch.int32, pin_memory=self.pin_memory + self.max_num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY ) self.num_computed_tokens = torch.zeros( self.max_num_reqs, dtype=torch.int32, device=self.device @@ -846,7 +846,7 @@ class GPUModelRunner( and self.speculative_config.use_ngram_gpu() ): self._num_valid_draft_tokens_cpu = torch.empty( - self.max_num_reqs, dtype=torch.int32, pin_memory=self.pin_memory + self.max_num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY ) self._num_valid_draft_tokens_event = torch.cuda.Event() self._num_valid_draft_tokens_copy_stream = torch.cuda.Stream() @@ -857,7 +857,7 @@ class GPUModelRunner( (self.max_num_reqs, 1), dtype=torch.int64, device="cpu", - pin_memory=self.pin_memory, + pin_memory=PIN_MEMORY, ) # Pre-allocated tensor for copying valid sampled token counts to CPU, @@ -879,7 +879,7 @@ class GPUModelRunner( (self.max_num_reqs, self.num_spec_tokens), dtype=torch.int64, device="cpu", - pin_memory=self.pin_memory, + pin_memory=PIN_MEMORY, ) if self.use_async_scheduling: self.valid_sampled_token_count_event = torch.Event() @@ -888,7 +888,7 @@ class GPUModelRunner( self.max_num_reqs, dtype=torch.int32, device="cpu", - pin_memory=self.pin_memory, + pin_memory=PIN_MEMORY, ) # Model weight offloader @@ -1000,7 +1000,6 @@ class GPUModelRunner( *size, dtype=dtype, device=self.device, - pin_memory=self.pin_memory, with_numpy=numpy, ) @@ -1055,7 +1054,7 @@ class GPUModelRunner( token_type_ids.append(ids) token_type_ids_cpu = torch.empty( - sum(seq_lens_cpu), dtype=torch.int32, pin_memory=self.pin_memory + sum(seq_lens_cpu), dtype=torch.int32, pin_memory=PIN_MEMORY ) torch.cat(token_type_ids, out=token_type_ids_cpu) model_kwargs["token_type_ids"] = token_type_ids_cpu.to( @@ -1095,12 +1094,12 @@ class GPUModelRunner( """ self._kv_block_zeroer = KVBlockZeroer( self.device, - self.pin_memory, + pin_memory=PIN_MEMORY, attn_groups_iter=self._kv_cache_spec_attn_group_iterator(), kernel_block_sizes=self._kernel_block_sizes, cache_dtype=self.cache_config.cache_dtype, runner_only_attn_layers=self.runner_only_attn_layers, - static_forward_context=(self.compilation_config.static_forward_context), + static_forward_context=self.compilation_config.static_forward_context, ) def _zero_block_ids(self, block_ids: list[int]) -> None: @@ -1651,7 +1650,7 @@ class GPUModelRunner( for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs( mm_kwargs, device=self.device, - pin_memory=self.pin_memory, + pin_memory=PIN_MEMORY, ): mm_kwargs_combined.update(mm_kwargs_batch) @@ -1807,10 +1806,10 @@ class GPUModelRunner( return # Upload the index tensors asynchronously so the scatter can be non-blocking. sampled_tokens_index_tensor = torch.tensor( - sample_flattened_indices, dtype=torch.int64, pin_memory=self.pin_memory + sample_flattened_indices, dtype=torch.int64, pin_memory=PIN_MEMORY ).to(self.device, non_blocking=True) prev_common_req_indices_tensor = torch.tensor( - prev_indices, dtype=torch.int64, pin_memory=self.pin_memory + prev_indices, dtype=torch.int64, pin_memory=PIN_MEMORY ).to(self.device, non_blocking=True) self.input_ids.gpu.scatter_( dim=0, @@ -1826,10 +1825,10 @@ class GPUModelRunner( assert isinstance(self._draft_token_ids, torch.Tensor) draft_tokens_index_tensor = torch.tensor( - spec_flattened_indices, dtype=torch.int64, pin_memory=self.pin_memory + spec_flattened_indices, dtype=torch.int64, pin_memory=PIN_MEMORY ).to(self.device, non_blocking=True) prev_draft_token_indices_tensor = torch.tensor( - prev_draft_token_indices, dtype=torch.int64, pin_memory=self.pin_memory + prev_draft_token_indices, dtype=torch.int64, pin_memory=PIN_MEMORY ).to(self.device, non_blocking=True) # because input_ids dtype is torch.int32, @@ -2788,21 +2787,16 @@ class GPUModelRunner( # [0, 1, 2, 5, 6, 9] target_logits_indices += self._arange_scratch[: cu_num_draft_tokens[-1]] - # TODO: Optimize the CPU -> GPU copy. - cu_num_draft_tokens = torch.from_numpy(cu_num_draft_tokens).to( - self.device, non_blocking=True + cu_num_draft_tokens = async_tensor_h2d(cu_num_draft_tokens, device=self.device) + cu_num_sampled_tokens = async_tensor_h2d( + cu_num_sampled_tokens, device=self.device ) - cu_num_sampled_tokens = torch.from_numpy(cu_num_sampled_tokens).to( - self.device, non_blocking=True + logits_indices = async_tensor_h2d(logits_indices, device=self.device) + target_logits_indices = async_tensor_h2d( + target_logits_indices, device=self.device ) - logits_indices = torch.from_numpy(logits_indices).to( - self.device, non_blocking=True - ) - target_logits_indices = torch.from_numpy(target_logits_indices).to( - self.device, non_blocking=True - ) - bonus_logits_indices = torch.from_numpy(bonus_logits_indices).to( - self.device, non_blocking=True + bonus_logits_indices = async_tensor_h2d( + bonus_logits_indices, device=self.device ) # Compute the draft token ids. @@ -3012,9 +3006,7 @@ class GPUModelRunner( # Track the current index in mm_kwargs/mm_lora_refs to map groups to request IDs current_item_idx = 0 for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs( - mm_kwargs, - device=self.device, - pin_memory=self.pin_memory, + mm_kwargs, device=self.device, pin_memory=PIN_MEMORY ): batch_outputs: MultiModalEmbeddings @@ -3048,7 +3040,7 @@ class GPUModelRunner( group_and_batch_mm_kwargs( [video_mm_kwargs_item], device=self.device, - pin_memory=self.pin_memory, + pin_memory=PIN_MEMORY, ) ) @@ -3107,7 +3099,10 @@ class GPUModelRunner( mm_embeds = list[torch.Tensor]() is_mm_embed = torch.zeros( - total_num_scheduled_tokens, dtype=torch.bool, device="cpu" + total_num_scheduled_tokens, + dtype=torch.bool, + device="cpu", + pin_memory=PIN_MEMORY, ) req_start_idx = 0 @@ -3515,8 +3510,7 @@ class GPUModelRunner( token_ids_idx_np = np.nonzero(is_token_ids)[0] # Some tokens ids may need to become embeds if token_ids_idx_np.size > 0: - token_ids_idx = torch.from_numpy(token_ids_idx_np) - token_ids_idx = token_ids_idx.to(self.device, non_blocking=True) + token_ids_idx = async_tensor_h2d(token_ids_idx_np, device=self.device) token_ids = self.input_ids.gpu[token_ids_idx] tokens_to_embeds = self.model.embed_input_ids(input_ids=token_ids) self.inputs_embeds.gpu[token_ids_idx] = tokens_to_embeds @@ -4953,7 +4947,7 @@ class GPUModelRunner( ): indices.append(offset + len(tokens) - 1) offset += num_draft + 1 - indices = torch.tensor(indices, device=self.device) + indices = async_tensor_h2d(indices, device=self.device) hidden_states = sample_hidden_states[indices] draft_token_ids = self.drafter.propose( @@ -5483,8 +5477,8 @@ class GPUModelRunner( continue num_prompt_tokens = len(request.prompt_token_ids) - prompt_token_ids = torch.tensor(request.prompt_token_ids).to( - self.device, non_blocking=True + prompt_token_ids = async_tensor_h2d( + request.prompt_token_ids, device=self.device ) # Set up target LogprobsTensors object. @@ -5651,7 +5645,7 @@ class GPUModelRunner( for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs( [(modality, dummy_mm_item)] * max_items_per_batch, device=self.device, - pin_memory=self.pin_memory, + pin_memory=PIN_MEMORY, ) ) @@ -6995,7 +6989,6 @@ class GPUModelRunner( max_model_len=max_model_len, max_num_batched_tokens=self.max_num_tokens, device=self.device, - pin_memory=self.pin_memory, vocab_size=self.model_config.get_vocab_size(), block_sizes=block_sizes, kernel_block_sizes=kernel_block_sizes, @@ -7430,7 +7423,7 @@ class GPUModelRunner( self.routed_experts_capturer.device_buffer.shape, dtype=self.routed_experts_capturer.device_buffer.dtype, device="cpu", - pin_memory=self.pin_memory, + pin_memory=PIN_MEMORY, ) # ``slot_mapping`` dtype is fixed to int64 by # ``block_table.slot_mapping``; we mirror that here. @@ -7439,7 +7432,7 @@ class GPUModelRunner( (max_tokens,), dtype=torch.int64, device="cpu", - pin_memory=self.pin_memory, + pin_memory=PIN_MEMORY, ) # Private device buffer so the shared ``block_table.slot_mapping`` # can be overwritten by the next ``_prepare_inputs`` while the