[MRV2] Buffer util simplifications (#56888)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-09-15 20:36:51 +00:00
committed by GitHub
parent b6ce714354
commit f2aad6aa70
21 changed files with 125 additions and 112 deletions
+25
View File
@@ -1,10 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
from vllm.utils.torch_utils import (
OMP_NUM_THREADS_SET_BY_VLLM,
async_tensor_h2d,
available_cpu_count,
common_broadcastable_dtype,
current_stream,
@@ -192,3 +194,26 @@ def test_runtime_threads_override_vllm_set_omp_num_threads(
torch.set_num_threads(3)
set_torch_threads_for_runtime()
assert torch.get_num_threads() == 1
@pytest.mark.parametrize(
"device", ["cpu", "cuda"] if torch.cuda.is_available() else ["cpu"]
)
def test_async_tensor_h2d_staging(device):
"""Staging must preserve values while normalizing dtype and layout."""
# Non-contiguous source with a dtype conversion: would previously sync.
src = torch.arange(12, dtype=torch.int64).reshape(3, 4).T
result = async_tensor_h2d(src, device=device, dtype=torch.int32)
assert result.dtype == torch.int32
assert result.is_contiguous()
assert torch.equal(result.cpu(), src.to(torch.int32))
# numpy and list sources still work.
assert torch.equal(
async_tensor_h2d(np.arange(4, dtype=np.int64), device=device),
torch.arange(4, device=device),
)
assert torch.equal(
async_tensor_h2d([1, 2, 3], device=device, dtype=torch.int32),
torch.tensor([1, 2, 3], dtype=torch.int32, device=device),
)
@@ -996,7 +996,7 @@ def test_pcp_plan_deinterleave_restores_global_order(
monkeypatch.setattr("vllm.v1.attention.backends.mla.indexer.PIN_MEMORY", False)
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.indexer.async_copy_to_gpu",
"vllm.v1.attention.backends.mla.indexer.async_tensor_h2d",
lambda x, device: x.to(device),
)
scheduled = np.array(req_lens, dtype=np.int64)
+3 -3
View File
@@ -17,7 +17,7 @@ from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers, set_dummy_c
from vllm.v1.worker.gpu.pcp_manager import PCPManager
def _copy_to_cpu(value, out=None, device=None):
def _copy_to_cpu(value, device=None, dtype=None, out=None):
tensor = torch.from_numpy(value) if isinstance(value, np.ndarray) else value
if out is not None:
return out.copy_(tensor)
@@ -83,7 +83,7 @@ def test_replicated_decode_piecewise_graph_padding(monkeypatch):
device=torch.device("cpu"),
dcp_world_size=1,
)
monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu)
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
segments_by_rank, per_rank_num_tokens = manager._build_batch_layout(
num_scheduled_tokens=np.ones(3, dtype=np.int32),
@@ -158,7 +158,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
device=torch.device("cpu"),
dcp_world_size=1,
)
monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu)
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
with pytest.raises(ValueError, match="smaller than the largest rank-local batch"):
manager._build_batch_layout(
@@ -269,11 +269,11 @@ from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
async_tensor_h2d,
direct_register_custom_op,
get_dtype_size,
is_quantized_kv_cache,
kv_cache_dtype_str_to_dtype,
np_to_pinned_tensor,
)
from vllm.v1.attention.backend import (
AttentionBackend,
@@ -1980,11 +1980,6 @@ def plan_mla_context_chunks(
return plans
def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor:
"""Pinned int32 CPU tensor backing one concatenated per-chunk field."""
return np_to_pinned_tensor(np.asarray(values, dtype=np.int32))
def align_mla_chunked_context_workspace_size(
vllm_config: VllmConfig,
workspace_size: int,
@@ -2161,24 +2156,28 @@ def build_mla_chunked_context_metadata(
token_offset += num_tokens
local_token_offset += num_local_tokens
seq_lens_cpu = _flat_int32(seq_lens_flat)
starts_and_context_lens = _flat_int32(starts_flat + context_lens).to(
device, non_blocking=True
seq_lens_cpu = torch.tensor(
seq_lens_flat, dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY
)
starts_and_context_lens = async_tensor_h2d(
starts_flat + context_lens, device=device, dtype=torch.int32
)
starts = starts_and_context_lens[: len(starts_flat)]
context_lens_gpu = starts_and_context_lens[len(starts_flat) :]
cu_seq_lens = _flat_int32(cu_seq_lens_flat).to(device, non_blocking=True)
cu_seqlens_q = _flat_int32(cu_seqlens_q_flat).to(device, non_blocking=True)
token_to_seq = _flat_int32(np.concatenate(token_to_seq_parts)).to(
device, non_blocking=True
cu_seq_lens = async_tensor_h2d(cu_seq_lens_flat, device=device, dtype=torch.int32)
cu_seqlens_q = async_tensor_h2d(cu_seqlens_q_flat, device=device, dtype=torch.int32)
token_to_seq = async_tensor_h2d(
np.concatenate(token_to_seq_parts), device=device, dtype=torch.int32
)
if use_dcp:
padded_local_cu_seq_lens = _flat_int32(padded_local_cu_seq_lens_flat).to(
device, non_blocking=True
padded_local_cu_seq_lens = async_tensor_h2d(
padded_local_cu_seq_lens_flat, device=device, dtype=torch.int32
)
padded_local_token_to_seq = async_tensor_h2d(
np.concatenate(padded_local_token_to_seq_parts),
device=device,
dtype=torch.int32,
)
padded_local_token_to_seq = _flat_int32(
np.concatenate(padded_local_token_to_seq_parts)
).to(device, non_blocking=True)
chunks: list[MLACommonPrefillMetadata.ContextChunk] = []
for index, (plan, layout) in enumerate(zip(plans, layouts)):
@@ -53,7 +53,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor, async_copy_to_gpu
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
@@ -1159,7 +1159,7 @@ class DiffusionSampler:
return
# Move the slot indices across once, up front: indexing a device
# tensor with a numpy array copies them over synchronously each time.
ps_gpu = async_copy_to_gpu(
ps_gpu = async_tensor_h2d(
ps.astype(np.int64), device=states.is_encoder_phase.device
)
states.init_canvas(ps_gpu)
@@ -1266,7 +1266,7 @@ class DiffusionSampler:
# was truncated near max_model_len, in which case the scheduler gave us
# fewer than CL logits for that request.
valid_canvas_len_np = per_req_nlogits_np[per_req_nlogits_np > 0]
valid_canvas_len = async_copy_to_gpu(
valid_canvas_len = async_tensor_h2d(
valid_canvas_len_np.astype(np.int64), device=device
)
+15 -7
View File
@@ -19,7 +19,11 @@ from vllm.model_executor.layers.attention.mla_attention import (
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.utils.torch_utils import (
PIN_MEMORY,
async_tensor_h2d,
is_quantized_kv_cache,
)
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionLayer,
@@ -231,8 +235,12 @@ def _build_sliding_window_metadata(
query_lens = query_lens_cpu[req_start:req_end]
kv_lens = kv_lens_cpu[req_start:req_end]
num_reqs = req_end - req_start
cu_seq_lens_q_cpu = torch.zeros(num_reqs + 1, dtype=torch.int32)
cu_seq_lens_k_cpu = torch.zeros(num_reqs + 1, dtype=torch.int32)
cu_seq_lens_q_cpu = torch.zeros(
num_reqs + 1, dtype=torch.int32, pin_memory=PIN_MEMORY
)
cu_seq_lens_k_cpu = torch.zeros(
num_reqs + 1, dtype=torch.int32, pin_memory=PIN_MEMORY
)
torch.cumsum(query_lens, 0, out=cu_seq_lens_q_cpu[1:])
torch.cumsum(kv_lens, 0, out=cu_seq_lens_k_cpu[1:])
token_to_seq_cpu = torch.repeat_interleave(
@@ -248,8 +256,8 @@ def _build_sliding_window_metadata(
query_end=query_end,
cu_seq_lens_q=cu_seq_lens_q_cpu.to(device, non_blocking=True),
cu_seq_lens_k=cu_seq_lens_k_cpu.to(device, non_blocking=True),
starts=starts_cpu[req_start:req_end].to(device, non_blocking=True),
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
starts=async_tensor_h2d(starts_cpu[req_start:req_end], device),
token_to_seq=async_tensor_h2d(token_to_seq_cpu, device),
num_kv_tokens=num_kv_tokens,
max_seq_len_q=int(query_lens.max().item()),
max_seq_len_k=int(kv_lens.max().item()),
@@ -396,8 +404,8 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder):
seq_lens = common_attn_metadata.seq_lens[
reqs_start + req_start : reqs_start + req_end
]
query_lens = query_lens_cpu[req_start:req_end].to(
self.device, non_blocking=True
query_lens = async_tensor_h2d(
query_lens_cpu[req_start:req_end], self.device
)
kv_lens = torch.minimum(seq_lens, query_lens + self.sliding_window - 1)
chunk.cu_seq_lens_k.zero_()
+3 -2
View File
@@ -10,6 +10,7 @@
import torch
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.triton_utils import triton
from .utils import tensor_cache
@@ -27,8 +28,8 @@ def prepare_chunk_indices(cu_seqlens: torch.Tensor, chunk_size: int) -> torch.Te
chunk_counts = triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist()
indices = torch.cat([torch.arange(n) for n in chunk_counts])
chunk_indices = torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1)
return chunk_indices.to(
device=cu_seqlens.device, dtype=cu_seqlens.dtype, non_blocking=True
return async_tensor_h2d(
chunk_indices, device=cu_seqlens.device, dtype=cu_seqlens.dtype
)
+15 -2
View File
@@ -705,17 +705,30 @@ def create_kv_caches_with_random(
def async_tensor_h2d(
data: list | np.ndarray | torch.Tensor,
device: str | torch.device,
device: str | torch.device | None = None,
dtype: torch.dtype | None = None,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Copy list/numpy array/tensor async from host to device."""
if dtype is None and out is not None:
dtype = out.dtype
if isinstance(data, np.ndarray):
data = torch.from_numpy(data)
if isinstance(data, torch.Tensor):
t = data.pin_memory() if PIN_MEMORY else data
t = data
if PIN_MEMORY and not t.is_pinned():
# Stage in pinned, contiguous buffer to ensure fully async copy.
t = torch.empty(
t.shape, dtype=dtype or t.dtype, device="cpu", pin_memory=True
).copy_(t)
else:
t = torch.tensor(data, dtype=dtype, pin_memory=PIN_MEMORY, device="cpu")
assert t.is_cpu
if out is not None:
assert out.dtype == dtype
return out.copy_(t, non_blocking=True)
assert device is not None, "must provide destination tensor or device"
return t.to(device=device, dtype=dtype, non_blocking=True)
+3 -14
View File
@@ -1264,15 +1264,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
"""
# write self.paged_kv_indptr_cpu inplace (0-index is always 0)
np.cumsum(
num_blocks_np,
dtype=np.int32,
out=self.paged_kv_indptr.np[1 : num_reqs + 1],
num_blocks_np, dtype=np.int32, out=self.paged_kv_indptr.np[1 : num_reqs + 1]
)
paged_kv_indptr = self.paged_kv_indptr.gpu[: num_reqs + 1]
paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_reqs + 1]
if PIN_MEMORY:
paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory()
paged_kv_indptr.copy_(paged_kv_indptr_cpu, non_blocking=True)
paged_kv_indptr = self.paged_kv_indptr.copy_to_gpu(num_reqs + 1)
# write self.paged_kv_indices inplace
num_actual_pages = self.paged_kv_indptr.np[num_reqs]
@@ -1292,12 +1286,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
page_size,
paged_kv_last_page_len_np,
)
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[:num_reqs]
if PIN_MEMORY:
paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory()
self.paged_kv_last_page_len.gpu[:num_reqs].copy_(
paged_kv_last_page_len_cpu, non_blocking=True
)
self.paged_kv_last_page_len.copy_to_gpu(num_reqs)
return paged_kv_indices
def build(
@@ -20,7 +20,11 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import current_stream, is_quantized_kv_cache
from vllm.utils.torch_utils import (
async_tensor_h2d,
current_stream,
is_quantized_kv_cache,
)
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -48,7 +52,6 @@ from vllm.v1.attention.ops.flashmla import (
get_mla_metadata,
)
from vllm.v1.kv_cache_interface import AttentionSpec
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
@@ -519,7 +522,7 @@ class FlashMLASparseMetadataBuilder(
)
workspace_rows = torch.from_numpy(rows_per_rank.astype(np.int32))
max_prefill_buffer_size //= self.dcp_world_size
entry_rows = async_copy_to_gpu(row_bounds[:-1], device=self.device)
entry_rows = async_tensor_h2d(row_bounds[:-1], device=self.device)
prefill_block_table = prefill_block_table.index_select(0, entry_rows)
prefill_seq_lens = prefill_seq_lens.index_select(0, entry_rows)
num_entries = len(workspace_rows)
+3 -4
View File
@@ -27,7 +27,7 @@ from vllm.utils.deep_gemm import (
)
from vllm.utils.math_utils import round_down
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -47,7 +47,6 @@ from vllm.v1.kv_cache_interface import (
KVCacheSpec,
MLAAttentionSpec,
)
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
logger = init_logger(__name__)
@@ -338,14 +337,14 @@ def build_pcp_global_chunk_plan(
+ local
)
cu = async_copy_to_gpu(cu, device=device)
cu = async_tensor_h2d(cu, device=device)
return PCPGlobalChunkPlan(
row_start_cu=cu[0],
global_cu=cu[1],
padded_local_cu=cu[2],
padded_local_total=padded_total,
total=total,
deinterleave_idx=async_copy_to_gpu(idx, device=device),
deinterleave_idx=async_tensor_h2d(idx, device=device),
)
+2 -3
View File
@@ -19,6 +19,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
triton_kernel_dispatcher_with_warmup,
)
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
from vllm.v1.sample.logits_processor.builtin import MinTokensLogitsProcessor
from vllm.v1.sample.metadata import SamplingMetadata
@@ -327,9 +328,7 @@ class RejectionSampler(nn.Module):
num_draft_tokens = torch.tensor(metadata.num_draft_tokens, device="cpu")
original_indices = torch.arange(num_requests, device="cpu")
repeat_indices_cpu = original_indices.repeat_interleave(num_draft_tokens)
repeat_indices = repeat_indices_cpu.to(
device=logits.device, non_blocking=True
)
repeat_indices = async_tensor_h2d(repeat_indices_cpu, logits.device)
logits = self.apply_penalties(
logits, sampling_metadata, metadata, repeat_indices, output_token_ids
)
-1
View File
@@ -255,7 +255,6 @@ class SpecDecodeBaseProposer:
self.backup_next_token_ids = CpuGpuBuffer(
self.max_batch_size,
dtype=torch.int32,
pin_memory=PIN_MEMORY,
device=device,
with_numpy=True,
)
+4 -3
View File
@@ -137,9 +137,10 @@ class CpuGpuBuffer:
self.np = self.cpu.numpy()
def copy_to_gpu(self, n: int | None = None) -> torch.Tensor:
if n is None:
return self.gpu.copy_(self.cpu, non_blocking=True)
return self.gpu[:n].copy_(self.cpu[:n], non_blocking=True)
cpu, gpu = self.cpu, self.gpu
if n is not None:
cpu, gpu = cpu[:n], gpu[:n]
return gpu.copy_(cpu.pin_memory() if PIN_MEMORY else cpu, non_blocking=True)
def copy_to_cpu(self, n: int | None = None) -> torch.Tensor:
"""NOTE: Because this method is non-blocking, explicit synchronization
+6 -3
View File
@@ -76,14 +76,17 @@ import vllm.utils.torch_utils as torch_utils
def async_tensor_h2d(
data: list | np.ndarray | torch.Tensor,
device: str | torch.device,
device: str | torch.device | None = None,
dtype: torch.dtype | None = None,
out: torch.Tensor | None = None,
) -> torch.Tensor:
if isinstance(data, np.ndarray):
data = torch.from_numpy(data)
if isinstance(data, torch.Tensor):
if not isinstance(data, torch.Tensor):
data = torch.tensor(data, dtype=dtype, device="cpu")
elif out is None:
return data.to(dtype=dtype)
return torch.tensor(data, dtype=dtype, device="cpu")
return data if out is None else out.copy_(data)
torch_utils.async_tensor_h2d = async_tensor_h2d
-19
View File
@@ -10,7 +10,6 @@ from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import (
PIN_MEMORY,
async_tensor_h2d,
get_accelerator_view_from_cpu_tensor,
)
@@ -27,24 +26,6 @@ def set_default_max_concurrency(n: int) -> None:
_DEFAULT_MAX_CONCURRENCY = max(2, n)
def async_copy_to_gpu(
x: torch.Tensor | np.ndarray,
out: torch.Tensor | None = None,
device: torch.device | None = None,
) -> torch.Tensor:
if isinstance(x, np.ndarray):
x = torch.from_numpy(x)
assert x.is_cpu
if out is None:
assert device is not None
out = torch.empty_like(x, device=device)
# pin_memory() is no-op if the memory is already pinned.
pinned = x.pin_memory() if PIN_MEMORY else x
return out.copy_(pinned, non_blocking=True)
class UvaBuffer:
def __init__(self, size: int | Sequence[int], dtype: torch.dtype):
if not is_uva_available():
+5 -8
View File
@@ -64,7 +64,7 @@ from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, async_tensor_h2d
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
@@ -99,10 +99,7 @@ from vllm.v1.worker.gpu.attn_utils import (
init_kv_cache,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.buffer_utils import (
async_copy_to_gpu,
set_default_max_concurrency,
)
from vllm.v1.worker.gpu.buffer_utils import set_default_max_concurrency
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
@@ -1289,7 +1286,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_ids = batch_req_state.req_ids
num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens
idx_mapping_np = batch_req_state.idx_mapping_np
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
idx_mapping = async_tensor_h2d(idx_mapping_np, device=self.device)
num_reqs = len(req_ids)
# Get the number of draft tokens for each request.
@@ -1324,7 +1321,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32)
cu_num_logits_np[0] = 0
np.cumsum(num_logits, out=cu_num_logits_np[1:])
cu_num_logits = async_copy_to_gpu(cu_num_logits_np, device=self.device)
cu_num_logits = async_tensor_h2d(cu_num_logits_np, device=self.device)
adaptive_verification = (
self.adaptive_verification if num_draft_tokens_per_req is not None else None
@@ -1352,7 +1349,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Some attention backends like FA3 require query_start_loc to be non-decreasing.
query_start_loc_np[num_reqs + 1 :] = num_tokens
query_start_loc = self.input_buffers.query_start_loc
async_copy_to_gpu(query_start_loc_np, out=query_start_loc)
async_tensor_h2d(query_start_loc_np, out=query_start_loc)
if adaptive_verification is not None:
cu_num_logits, query_start_loc, total_num_draft_tokens = (
adaptive_verification.reallocate_drafts(req_ids, idx_mapping)
+6 -6
View File
@@ -10,9 +10,9 @@ import torch
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.parallel_state import get_dcp_group, get_pcp_group
from vllm.logger import init_logger
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.utils import PAD_SLOT_ID, get_dcp_local_seq_lens
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.v1.worker.gpu.input_batch import (
InputBatch,
InputBuffers,
@@ -358,13 +358,13 @@ class PCPManager:
dtype=np.int64,
)
self._hidden_restore_idx = async_copy_to_gpu(
self._hidden_restore_idx = async_tensor_h2d(
hidden_restore_idx, device=self.device
)
self._padded_gather_idx = async_copy_to_gpu(
self._padded_gather_idx = async_tensor_h2d(
padded_gather_idx, device=self.device
)
self._gathered_kv_write_mask = async_copy_to_gpu(
self._gathered_kv_write_mask = async_tensor_h2d(
gathered_kv_write_mask, device=self.device
)
return segments_by_rank, per_rank_num_tokens
@@ -536,12 +536,12 @@ class PCPManager:
local_query_start_loc_out = local_query_start_loc_np[1 : num_local_reqs + 1]
np.cumsum(local_num_scheduled_tokens, out=local_query_start_loc_out)
local_query_start_loc_np[num_local_reqs + 1 :] = num_local_tokens
async_copy_to_gpu(local_query_start_loc_np, out=input_buffers.query_start_loc)
async_tensor_h2d(local_query_start_loc_np, out=input_buffers.query_start_loc)
local_query_start_loc = input_buffers.query_start_loc[
: num_reqs_after_padding + 1
]
local_to_global_req_idx = async_copy_to_gpu(
local_to_global_req_idx = async_tensor_h2d(
local_to_global_req_idx_np, device=self.device
)
seq_lens = input_buffers.seq_lens[:num_reqs_after_padding]
+3 -3
View File
@@ -11,7 +11,7 @@ import torch
from vllm.distributed.parallel_state import get_pp_group
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.worker.gpu.input_batch import InputBatch
@@ -138,7 +138,7 @@ class PPHandler:
return None
# Filter excluded request indices.
idx_mapping_np = np.where(exclude_mask, -1, slot.idx_mapping_np)
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
idx_mapping = async_tensor_h2d(idx_mapping_np, device=self.device)
self.main_stream.wait_event(slot.event)
if slot.draft_tokens is not None and draft_tokens_to_update is not None:
@@ -148,7 +148,7 @@ class PPHandler:
keep = ~exclude_mask
keep_t = torch.as_tensor(keep, device=self.device)
draft_tokens = draft_tokens[keep_t]
draft_idx_mapping = async_copy_to_gpu(
draft_idx_mapping = async_tensor_h2d(
slot.idx_mapping_np[keep], device=self.device
)
draft_tokens_to_update[draft_idx_mapping] = draft_tokens
@@ -14,6 +14,7 @@ from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.parallel_state import get_tp_group
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.utils import CpuGpuBuffer
from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream
@@ -21,7 +22,6 @@ from vllm.v1.worker.gpu.attn_utils import (
get_attn_cg_support,
get_query_lens_mismatch_unsupported_backend,
)
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
logger = init_logger(__name__)
_PROFILE_REPLAYS = 5
@@ -413,7 +413,7 @@ class AdaptiveVerificationManager:
if draft_budget == 0:
capacities.zero_()
else:
async_copy_to_gpu(scheduled_drafts, out=capacities)
async_tensor_h2d(scheduled_drafts, out=capacities)
if draft_budget < int(scheduled_drafts.sum()):
_assign_draft_token_budget_compiled(
self._confidence_probs,
@@ -424,10 +424,7 @@ class AdaptiveVerificationManager:
)
num_non_draft_tokens_gpu = self._num_non_draft_tokens[:num_reqs]
async_copy_to_gpu(
num_non_draft_tokens,
out=num_non_draft_tokens_gpu,
)
async_tensor_h2d(num_non_draft_tokens, out=num_non_draft_tokens_gpu)
self._cu_num_logits[:1].zero_()
torch.cumsum(
capacities + self.num_bonus_tokens,
+2 -3
View File
@@ -5,8 +5,7 @@ import torch
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.worker.gpu.input_batch import InputBatch
@@ -68,7 +67,7 @@ class StructuredOutputsWorker:
# Asynchronously copy the bitmask to GPU.
with torch.cuda.stream(self.copy_stream):
bitmask = async_copy_to_gpu(
bitmask = async_tensor_h2d(
grammar_bitmask, out=self.grammar_bitmask[: grammar_bitmask.shape[0]]
)