diff --git a/tests/utils_/test_torch_utils.py b/tests/utils_/test_torch_utils.py index 6901a162a4..79dd3e320a 100644 --- a/tests/utils_/test_torch_utils.py +++ b/tests/utils_/test_torch_utils.py @@ -1,10 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import numpy as np import pytest import torch from vllm.utils.torch_utils import ( OMP_NUM_THREADS_SET_BY_VLLM, + async_tensor_h2d, available_cpu_count, common_broadcastable_dtype, current_stream, @@ -192,3 +194,26 @@ def test_runtime_threads_override_vllm_set_omp_num_threads( torch.set_num_threads(3) set_torch_threads_for_runtime() assert torch.get_num_threads() == 1 + + +@pytest.mark.parametrize( + "device", ["cpu", "cuda"] if torch.cuda.is_available() else ["cpu"] +) +def test_async_tensor_h2d_staging(device): + """Staging must preserve values while normalizing dtype and layout.""" + # Non-contiguous source with a dtype conversion: would previously sync. + src = torch.arange(12, dtype=torch.int64).reshape(3, 4).T + result = async_tensor_h2d(src, device=device, dtype=torch.int32) + assert result.dtype == torch.int32 + assert result.is_contiguous() + assert torch.equal(result.cpu(), src.to(torch.int32)) + + # numpy and list sources still work. + assert torch.equal( + async_tensor_h2d(np.arange(4, dtype=np.int64), device=device), + torch.arange(4, device=device), + ) + assert torch.equal( + async_tensor_h2d([1, 2, 3], device=device, dtype=torch.int32), + torch.tensor([1, 2, 3], dtype=torch.int32, device=device), + ) diff --git a/tests/v1/attention/test_indexer_dcp_localize.py b/tests/v1/attention/test_indexer_dcp_localize.py index 812bce39d8..8bc409adb7 100644 --- a/tests/v1/attention/test_indexer_dcp_localize.py +++ b/tests/v1/attention/test_indexer_dcp_localize.py @@ -996,7 +996,7 @@ def test_pcp_plan_deinterleave_restores_global_order( monkeypatch.setattr("vllm.v1.attention.backends.mla.indexer.PIN_MEMORY", False) monkeypatch.setattr( - "vllm.v1.attention.backends.mla.indexer.async_copy_to_gpu", + "vllm.v1.attention.backends.mla.indexer.async_tensor_h2d", lambda x, device: x.to(device), ) scheduled = np.array(req_lens, dtype=np.int64) diff --git a/tests/v1/worker/test_gpu_pcp_manager.py b/tests/v1/worker/test_gpu_pcp_manager.py index d3ec562aa7..2bd7be2cfa 100644 --- a/tests/v1/worker/test_gpu_pcp_manager.py +++ b/tests/v1/worker/test_gpu_pcp_manager.py @@ -17,7 +17,7 @@ from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers, set_dummy_c from vllm.v1.worker.gpu.pcp_manager import PCPManager -def _copy_to_cpu(value, out=None, device=None): +def _copy_to_cpu(value, device=None, dtype=None, out=None): tensor = torch.from_numpy(value) if isinstance(value, np.ndarray) else value if out is not None: return out.copy_(tensor) @@ -83,7 +83,7 @@ def test_replicated_decode_piecewise_graph_padding(monkeypatch): device=torch.device("cpu"), dcp_world_size=1, ) - monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu) + monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu) segments_by_rank, per_rank_num_tokens = manager._build_batch_layout( num_scheduled_tokens=np.ones(3, dtype=np.int32), @@ -158,7 +158,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch): device=torch.device("cpu"), dcp_world_size=1, ) - monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu) + monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu) with pytest.raises(ValueError, match="smaller than the largest rank-local batch"): manager._build_batch_layout( diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 874c43f8cd..1e673bd2da 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -269,11 +269,11 @@ from vllm.utils.torch_utils import ( LayerNameType, _encode_layer_name, _resolve_layer_name, + async_tensor_h2d, direct_register_custom_op, get_dtype_size, is_quantized_kv_cache, kv_cache_dtype_str_to_dtype, - np_to_pinned_tensor, ) from vllm.v1.attention.backend import ( AttentionBackend, @@ -1980,11 +1980,6 @@ def plan_mla_context_chunks( return plans -def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor: - """Pinned int32 CPU tensor backing one concatenated per-chunk field.""" - return np_to_pinned_tensor(np.asarray(values, dtype=np.int32)) - - def align_mla_chunked_context_workspace_size( vllm_config: VllmConfig, workspace_size: int, @@ -2161,24 +2156,28 @@ def build_mla_chunked_context_metadata( token_offset += num_tokens local_token_offset += num_local_tokens - seq_lens_cpu = _flat_int32(seq_lens_flat) - starts_and_context_lens = _flat_int32(starts_flat + context_lens).to( - device, non_blocking=True + seq_lens_cpu = torch.tensor( + seq_lens_flat, dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY + ) + starts_and_context_lens = async_tensor_h2d( + starts_flat + context_lens, device=device, dtype=torch.int32 ) starts = starts_and_context_lens[: len(starts_flat)] context_lens_gpu = starts_and_context_lens[len(starts_flat) :] - cu_seq_lens = _flat_int32(cu_seq_lens_flat).to(device, non_blocking=True) - cu_seqlens_q = _flat_int32(cu_seqlens_q_flat).to(device, non_blocking=True) - token_to_seq = _flat_int32(np.concatenate(token_to_seq_parts)).to( - device, non_blocking=True + cu_seq_lens = async_tensor_h2d(cu_seq_lens_flat, device=device, dtype=torch.int32) + cu_seqlens_q = async_tensor_h2d(cu_seqlens_q_flat, device=device, dtype=torch.int32) + token_to_seq = async_tensor_h2d( + np.concatenate(token_to_seq_parts), device=device, dtype=torch.int32 ) if use_dcp: - padded_local_cu_seq_lens = _flat_int32(padded_local_cu_seq_lens_flat).to( - device, non_blocking=True + padded_local_cu_seq_lens = async_tensor_h2d( + padded_local_cu_seq_lens_flat, device=device, dtype=torch.int32 + ) + padded_local_token_to_seq = async_tensor_h2d( + np.concatenate(padded_local_token_to_seq_parts), + device=device, + dtype=torch.int32, ) - padded_local_token_to_seq = _flat_int32( - np.concatenate(padded_local_token_to_seq_parts) - ).to(device, non_blocking=True) chunks: list[MLACommonPrefillMetadata.ContextChunk] = [] for index, (plan, layout) in enumerate(zip(plans, layouts)): diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 603d68410d..c90d78bd6d 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -53,7 +53,7 @@ from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.outputs import LogprobsTensors from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p from vllm.v1.worker.gpu.attn_utils import build_attn_metadata -from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor, async_copy_to_gpu +from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores @@ -1159,7 +1159,7 @@ class DiffusionSampler: return # Move the slot indices across once, up front: indexing a device # tensor with a numpy array copies them over synchronously each time. - ps_gpu = async_copy_to_gpu( + ps_gpu = async_tensor_h2d( ps.astype(np.int64), device=states.is_encoder_phase.device ) states.init_canvas(ps_gpu) @@ -1266,7 +1266,7 @@ class DiffusionSampler: # was truncated near max_model_len, in which case the scheduler gave us # fewer than CL logits for that request. valid_canvas_len_np = per_req_nlogits_np[per_req_nlogits_np > 0] - valid_canvas_len = async_copy_to_gpu( + valid_canvas_len = async_tensor_h2d( valid_canvas_len_np.astype(np.int64), device=device ) diff --git a/vllm/models/dots3_note/nvidia/attention.py b/vllm/models/dots3_note/nvidia/attention.py index cbd9ad7f4d..27928b6e6f 100644 --- a/vllm/models/dots3_note/nvidia/attention.py +++ b/vllm/models/dots3_note/nvidia/attention.py @@ -19,7 +19,11 @@ from vllm.model_executor.layers.attention.mla_attention import ( ) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton -from vllm.utils.torch_utils import is_quantized_kv_cache +from vllm.utils.torch_utils import ( + PIN_MEMORY, + async_tensor_h2d, + is_quantized_kv_cache, +) from vllm.v1.attention.backend import ( AttentionCGSupport, AttentionLayer, @@ -231,8 +235,12 @@ def _build_sliding_window_metadata( query_lens = query_lens_cpu[req_start:req_end] kv_lens = kv_lens_cpu[req_start:req_end] num_reqs = req_end - req_start - cu_seq_lens_q_cpu = torch.zeros(num_reqs + 1, dtype=torch.int32) - cu_seq_lens_k_cpu = torch.zeros(num_reqs + 1, dtype=torch.int32) + cu_seq_lens_q_cpu = torch.zeros( + num_reqs + 1, dtype=torch.int32, pin_memory=PIN_MEMORY + ) + cu_seq_lens_k_cpu = torch.zeros( + num_reqs + 1, dtype=torch.int32, pin_memory=PIN_MEMORY + ) torch.cumsum(query_lens, 0, out=cu_seq_lens_q_cpu[1:]) torch.cumsum(kv_lens, 0, out=cu_seq_lens_k_cpu[1:]) token_to_seq_cpu = torch.repeat_interleave( @@ -248,8 +256,8 @@ def _build_sliding_window_metadata( query_end=query_end, cu_seq_lens_q=cu_seq_lens_q_cpu.to(device, non_blocking=True), cu_seq_lens_k=cu_seq_lens_k_cpu.to(device, non_blocking=True), - starts=starts_cpu[req_start:req_end].to(device, non_blocking=True), - token_to_seq=token_to_seq_cpu.to(device, non_blocking=True), + starts=async_tensor_h2d(starts_cpu[req_start:req_end], device), + token_to_seq=async_tensor_h2d(token_to_seq_cpu, device), num_kv_tokens=num_kv_tokens, max_seq_len_q=int(query_lens.max().item()), max_seq_len_k=int(kv_lens.max().item()), @@ -396,8 +404,8 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder): seq_lens = common_attn_metadata.seq_lens[ reqs_start + req_start : reqs_start + req_end ] - query_lens = query_lens_cpu[req_start:req_end].to( - self.device, non_blocking=True + query_lens = async_tensor_h2d( + query_lens_cpu[req_start:req_end], self.device ) kv_lens = torch.minimum(seq_lens, query_lens + self.sliding_window - 1) chunk.cu_seq_lens_k.zero_() diff --git a/vllm/third_party/flash_linear_attention/ops/index.py b/vllm/third_party/flash_linear_attention/ops/index.py index cc6f54d8af..880be7747d 100644 --- a/vllm/third_party/flash_linear_attention/ops/index.py +++ b/vllm/third_party/flash_linear_attention/ops/index.py @@ -10,6 +10,7 @@ import torch from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import async_tensor_h2d from vllm.triton_utils import triton from .utils import tensor_cache @@ -27,8 +28,8 @@ def prepare_chunk_indices(cu_seqlens: torch.Tensor, chunk_size: int) -> torch.Te chunk_counts = triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist() indices = torch.cat([torch.arange(n) for n in chunk_counts]) chunk_indices = torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1) - return chunk_indices.to( - device=cu_seqlens.device, dtype=cu_seqlens.dtype, non_blocking=True + return async_tensor_h2d( + chunk_indices, device=cu_seqlens.device, dtype=cu_seqlens.dtype ) diff --git a/vllm/utils/torch_utils.py b/vllm/utils/torch_utils.py index 6f7dc14346..0b330edf00 100644 --- a/vllm/utils/torch_utils.py +++ b/vllm/utils/torch_utils.py @@ -705,17 +705,30 @@ def create_kv_caches_with_random( def async_tensor_h2d( data: list | np.ndarray | torch.Tensor, - device: str | torch.device, + device: str | torch.device | None = None, dtype: torch.dtype | None = None, + out: torch.Tensor | None = None, ) -> torch.Tensor: """Copy list/numpy array/tensor async from host to device.""" + if dtype is None and out is not None: + dtype = out.dtype if isinstance(data, np.ndarray): data = torch.from_numpy(data) if isinstance(data, torch.Tensor): - t = data.pin_memory() if PIN_MEMORY else data + t = data + if PIN_MEMORY and not t.is_pinned(): + # Stage in pinned, contiguous buffer to ensure fully async copy. + t = torch.empty( + t.shape, dtype=dtype or t.dtype, device="cpu", pin_memory=True + ).copy_(t) else: t = torch.tensor(data, dtype=dtype, pin_memory=PIN_MEMORY, device="cpu") assert t.is_cpu + + if out is not None: + assert out.dtype == dtype + return out.copy_(t, non_blocking=True) + assert device is not None, "must provide destination tensor or device" return t.to(device=device, dtype=dtype, non_blocking=True) diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 0da68e3e3d..7ab2b2018e 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -1264,15 +1264,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): """ # write self.paged_kv_indptr_cpu inplace (0-index is always 0) np.cumsum( - num_blocks_np, - dtype=np.int32, - out=self.paged_kv_indptr.np[1 : num_reqs + 1], + num_blocks_np, dtype=np.int32, out=self.paged_kv_indptr.np[1 : num_reqs + 1] ) - paged_kv_indptr = self.paged_kv_indptr.gpu[: num_reqs + 1] - paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_reqs + 1] - if PIN_MEMORY: - paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory() - paged_kv_indptr.copy_(paged_kv_indptr_cpu, non_blocking=True) + paged_kv_indptr = self.paged_kv_indptr.copy_to_gpu(num_reqs + 1) # write self.paged_kv_indices inplace num_actual_pages = self.paged_kv_indptr.np[num_reqs] @@ -1292,12 +1286,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): page_size, paged_kv_last_page_len_np, ) - paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[:num_reqs] - if PIN_MEMORY: - paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory() - self.paged_kv_last_page_len.gpu[:num_reqs].copy_( - paged_kv_last_page_len_cpu, non_blocking=True - ) + self.paged_kv_last_page_len.copy_to_gpu(num_reqs) return paged_kv_indices def build( diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 9c440776ae..3a9ca2ffdb 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -20,7 +20,11 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import ( from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.utils.platform_utils import num_compute_units -from vllm.utils.torch_utils import current_stream, is_quantized_kv_cache +from vllm.utils.torch_utils import ( + async_tensor_h2d, + current_stream, + is_quantized_kv_cache, +) from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -48,7 +52,6 @@ from vllm.v1.attention.ops.flashmla import ( get_mla_metadata, ) from vllm.v1.kv_cache_interface import AttentionSpec -from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu from vllm.v1.worker.workspace import current_workspace_manager if TYPE_CHECKING: @@ -519,7 +522,7 @@ class FlashMLASparseMetadataBuilder( ) workspace_rows = torch.from_numpy(rows_per_rank.astype(np.int32)) max_prefill_buffer_size //= self.dcp_world_size - entry_rows = async_copy_to_gpu(row_bounds[:-1], device=self.device) + entry_rows = async_tensor_h2d(row_bounds[:-1], device=self.device) prefill_block_table = prefill_block_table.index_select(0, entry_rows) prefill_seq_lens = prefill_seq_lens.index_select(0, entry_rows) num_entries = len(workspace_rows) diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 28bd450da1..837dbca817 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -27,7 +27,7 @@ from vllm.utils.deep_gemm import ( ) from vllm.utils.math_utils import round_down from vllm.utils.platform_utils import num_compute_units -from vllm.utils.torch_utils import PIN_MEMORY +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -47,7 +47,6 @@ from vllm.v1.kv_cache_interface import ( KVCacheSpec, MLAAttentionSpec, ) -from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu logger = init_logger(__name__) @@ -338,14 +337,14 @@ def build_pcp_global_chunk_plan( + local ) - cu = async_copy_to_gpu(cu, device=device) + cu = async_tensor_h2d(cu, device=device) return PCPGlobalChunkPlan( row_start_cu=cu[0], global_cu=cu[1], padded_local_cu=cu[2], padded_local_total=padded_total, total=total, - deinterleave_idx=async_copy_to_gpu(idx, device=device), + deinterleave_idx=async_tensor_h2d(idx, device=device), ) diff --git a/vllm/v1/sample/rejection_sampler.py b/vllm/v1/sample/rejection_sampler.py index 5aa8345473..85b623bd84 100644 --- a/vllm/v1/sample/rejection_sampler.py +++ b/vllm/v1/sample/rejection_sampler.py @@ -19,6 +19,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( triton_kernel_dispatcher_with_warmup, ) from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput from vllm.v1.sample.logits_processor.builtin import MinTokensLogitsProcessor from vllm.v1.sample.metadata import SamplingMetadata @@ -327,9 +328,7 @@ class RejectionSampler(nn.Module): num_draft_tokens = torch.tensor(metadata.num_draft_tokens, device="cpu") original_indices = torch.arange(num_requests, device="cpu") repeat_indices_cpu = original_indices.repeat_interleave(num_draft_tokens) - repeat_indices = repeat_indices_cpu.to( - device=logits.device, non_blocking=True - ) + repeat_indices = async_tensor_h2d(repeat_indices_cpu, logits.device) logits = self.apply_penalties( logits, sampling_metadata, metadata, repeat_indices, output_token_ids ) diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index c0e456521c..1bd1bdc45b 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -255,7 +255,6 @@ class SpecDecodeBaseProposer: self.backup_next_token_ids = CpuGpuBuffer( self.max_batch_size, dtype=torch.int32, - pin_memory=PIN_MEMORY, device=device, with_numpy=True, ) diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index da17110d80..facafd2860 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -137,9 +137,10 @@ class CpuGpuBuffer: self.np = self.cpu.numpy() def copy_to_gpu(self, n: int | None = None) -> torch.Tensor: - if n is None: - return self.gpu.copy_(self.cpu, non_blocking=True) - return self.gpu[:n].copy_(self.cpu[:n], non_blocking=True) + cpu, gpu = self.cpu, self.gpu + if n is not None: + cpu, gpu = cpu[:n], gpu[:n] + return gpu.copy_(cpu.pin_memory() if PIN_MEMORY else cpu, non_blocking=True) def copy_to_cpu(self, n: int | None = None) -> torch.Tensor: """NOTE: Because this method is non-blocking, explicit synchronization diff --git a/vllm/v1/worker/cpu/shm.py b/vllm/v1/worker/cpu/shm.py index f92c6c3bf2..b615f4b5b5 100644 --- a/vllm/v1/worker/cpu/shm.py +++ b/vllm/v1/worker/cpu/shm.py @@ -76,14 +76,17 @@ import vllm.utils.torch_utils as torch_utils def async_tensor_h2d( data: list | np.ndarray | torch.Tensor, - device: str | torch.device, + device: str | torch.device | None = None, dtype: torch.dtype | None = None, + out: torch.Tensor | None = None, ) -> torch.Tensor: if isinstance(data, np.ndarray): data = torch.from_numpy(data) - if isinstance(data, torch.Tensor): + if not isinstance(data, torch.Tensor): + data = torch.tensor(data, dtype=dtype, device="cpu") + elif out is None: return data.to(dtype=dtype) - return torch.tensor(data, dtype=dtype, device="cpu") + return data if out is None else out.copy_(data) torch_utils.async_tensor_h2d = async_tensor_h2d diff --git a/vllm/v1/worker/gpu/buffer_utils.py b/vllm/v1/worker/gpu/buffer_utils.py index 0066f80907..0731072e59 100644 --- a/vllm/v1/worker/gpu/buffer_utils.py +++ b/vllm/v1/worker/gpu/buffer_utils.py @@ -10,7 +10,6 @@ from vllm.logger import init_logger from vllm.triton_utils import tl, triton from vllm.utils.platform_utils import is_uva_available from vllm.utils.torch_utils import ( - PIN_MEMORY, async_tensor_h2d, get_accelerator_view_from_cpu_tensor, ) @@ -27,24 +26,6 @@ def set_default_max_concurrency(n: int) -> None: _DEFAULT_MAX_CONCURRENCY = max(2, n) -def async_copy_to_gpu( - x: torch.Tensor | np.ndarray, - out: torch.Tensor | None = None, - device: torch.device | None = None, -) -> torch.Tensor: - if isinstance(x, np.ndarray): - x = torch.from_numpy(x) - assert x.is_cpu - - if out is None: - assert device is not None - out = torch.empty_like(x, device=device) - - # pin_memory() is no-op if the memory is already pinned. - pinned = x.pin_memory() if PIN_MEMORY else x - return out.copy_(pinned, non_blocking=True) - - class UvaBuffer: def __init__(self, size: int | Sequence[int], dtype: torch.dtype): if not is_uva_available(): diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 25395c3330..b66d552cb4 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -64,7 +64,7 @@ from vllm.sequence import IntermediateTensors from vllm.tasks import SupportedTask from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib -from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE +from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, async_tensor_h2d from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.kv_cache_interface import ( KVCacheConfig, @@ -99,10 +99,7 @@ from vllm.v1.worker.gpu.attn_utils import ( init_kv_cache, ) from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.buffer_utils import ( - async_copy_to_gpu, - set_default_max_concurrency, -) +from vllm.v1.worker.gpu.buffer_utils import set_default_max_concurrency from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.cudagraph_utils import ( BatchExecutionDescriptor, @@ -1289,7 +1286,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): req_ids = batch_req_state.req_ids num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens idx_mapping_np = batch_req_state.idx_mapping_np - idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device) + idx_mapping = async_tensor_h2d(idx_mapping_np, device=self.device) num_reqs = len(req_ids) # Get the number of draft tokens for each request. @@ -1324,7 +1321,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32) cu_num_logits_np[0] = 0 np.cumsum(num_logits, out=cu_num_logits_np[1:]) - cu_num_logits = async_copy_to_gpu(cu_num_logits_np, device=self.device) + cu_num_logits = async_tensor_h2d(cu_num_logits_np, device=self.device) adaptive_verification = ( self.adaptive_verification if num_draft_tokens_per_req is not None else None @@ -1352,7 +1349,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Some attention backends like FA3 require query_start_loc to be non-decreasing. query_start_loc_np[num_reqs + 1 :] = num_tokens query_start_loc = self.input_buffers.query_start_loc - async_copy_to_gpu(query_start_loc_np, out=query_start_loc) + async_tensor_h2d(query_start_loc_np, out=query_start_loc) if adaptive_verification is not None: cu_num_logits, query_start_loc, total_num_draft_tokens = ( adaptive_verification.reallocate_drafts(req_ids, idx_mapping) diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index adfbf2f9f0..376e0f2510 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -10,9 +10,9 @@ import torch from vllm.config import CUDAGraphMode, VllmConfig from vllm.distributed.parallel_state import get_dcp_group, get_pcp_group from vllm.logger import init_logger +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backends.utils import PAD_SLOT_ID, get_dcp_local_seq_lens from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu from vllm.v1.worker.gpu.input_batch import ( InputBatch, InputBuffers, @@ -358,13 +358,13 @@ class PCPManager: dtype=np.int64, ) - self._hidden_restore_idx = async_copy_to_gpu( + self._hidden_restore_idx = async_tensor_h2d( hidden_restore_idx, device=self.device ) - self._padded_gather_idx = async_copy_to_gpu( + self._padded_gather_idx = async_tensor_h2d( padded_gather_idx, device=self.device ) - self._gathered_kv_write_mask = async_copy_to_gpu( + self._gathered_kv_write_mask = async_tensor_h2d( gathered_kv_write_mask, device=self.device ) return segments_by_rank, per_rank_num_tokens @@ -536,12 +536,12 @@ class PCPManager: local_query_start_loc_out = local_query_start_loc_np[1 : num_local_reqs + 1] np.cumsum(local_num_scheduled_tokens, out=local_query_start_loc_out) local_query_start_loc_np[num_local_reqs + 1 :] = num_local_tokens - async_copy_to_gpu(local_query_start_loc_np, out=input_buffers.query_start_loc) + async_tensor_h2d(local_query_start_loc_np, out=input_buffers.query_start_loc) local_query_start_loc = input_buffers.query_start_loc[ : num_reqs_after_padding + 1 ] - local_to_global_req_idx = async_copy_to_gpu( + local_to_global_req_idx = async_tensor_h2d( local_to_global_req_idx_np, device=self.device ) seq_lens = input_buffers.seq_lens[:num_reqs_after_padding] diff --git a/vllm/v1/worker/gpu/pp_utils.py b/vllm/v1/worker/gpu/pp_utils.py index 8c65a23ef7..ab1b991d1a 100644 --- a/vllm/v1/worker/gpu/pp_utils.py +++ b/vllm/v1/worker/gpu/pp_utils.py @@ -11,7 +11,7 @@ import torch from vllm.distributed.parallel_state import get_pp_group from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.worker.gpu.input_batch import InputBatch @@ -138,7 +138,7 @@ class PPHandler: return None # Filter excluded request indices. idx_mapping_np = np.where(exclude_mask, -1, slot.idx_mapping_np) - idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device) + idx_mapping = async_tensor_h2d(idx_mapping_np, device=self.device) self.main_stream.wait_event(slot.event) if slot.draft_tokens is not None and draft_tokens_to_update is not None: @@ -148,7 +148,7 @@ class PPHandler: keep = ~exclude_mask keep_t = torch.as_tensor(keep, device=self.device) draft_tokens = draft_tokens[keep_t] - draft_idx_mapping = async_copy_to_gpu( + draft_idx_mapping = async_tensor_h2d( slot.idx_mapping_np[keep], device=self.device ) draft_tokens_to_update[draft_idx_mapping] = draft_tokens diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index af58b8993b..245f304ac4 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -14,6 +14,7 @@ from vllm.config.compilation import CUDAGraphMode from vllm.distributed.parallel_state import get_tp_group from vllm.logger import init_logger from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.utils import CpuGpuBuffer from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream @@ -21,7 +22,6 @@ from vllm.v1.worker.gpu.attn_utils import ( get_attn_cg_support, get_query_lens_mismatch_unsupported_backend, ) -from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu logger = init_logger(__name__) _PROFILE_REPLAYS = 5 @@ -413,7 +413,7 @@ class AdaptiveVerificationManager: if draft_budget == 0: capacities.zero_() else: - async_copy_to_gpu(scheduled_drafts, out=capacities) + async_tensor_h2d(scheduled_drafts, out=capacities) if draft_budget < int(scheduled_drafts.sum()): _assign_draft_token_budget_compiled( self._confidence_probs, @@ -424,10 +424,7 @@ class AdaptiveVerificationManager: ) num_non_draft_tokens_gpu = self._num_non_draft_tokens[:num_reqs] - async_copy_to_gpu( - num_non_draft_tokens, - out=num_non_draft_tokens_gpu, - ) + async_tensor_h2d(num_non_draft_tokens, out=num_non_draft_tokens_gpu) self._cu_num_logits[:1].zero_() torch.cumsum( capacities + self.num_bonus_tokens, diff --git a/vllm/v1/worker/gpu/structured_outputs.py b/vllm/v1/worker/gpu/structured_outputs.py index 8437b0aed9..057d6f7154 100644 --- a/vllm/v1/worker/gpu/structured_outputs.py +++ b/vllm/v1/worker/gpu/structured_outputs.py @@ -5,8 +5,7 @@ import torch from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv -from vllm.utils.torch_utils import PIN_MEMORY -from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from vllm.v1.worker.gpu.input_batch import InputBatch @@ -68,7 +67,7 @@ class StructuredOutputsWorker: # Asynchronously copy the bitmask to GPU. with torch.cuda.stream(self.copy_stream): - bitmask = async_copy_to_gpu( + bitmask = async_tensor_h2d( grammar_bitmask, out=self.grammar_bitmask[: grammar_bitmask.shape[0]] )