[MRV2] Buffer util simplifications (#56888)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import (
|
||||
OMP_NUM_THREADS_SET_BY_VLLM,
|
||||
async_tensor_h2d,
|
||||
available_cpu_count,
|
||||
common_broadcastable_dtype,
|
||||
current_stream,
|
||||
@@ -192,3 +194,26 @@ def test_runtime_threads_override_vllm_set_omp_num_threads(
|
||||
torch.set_num_threads(3)
|
||||
set_torch_threads_for_runtime()
|
||||
assert torch.get_num_threads() == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"device", ["cpu", "cuda"] if torch.cuda.is_available() else ["cpu"]
|
||||
)
|
||||
def test_async_tensor_h2d_staging(device):
|
||||
"""Staging must preserve values while normalizing dtype and layout."""
|
||||
# Non-contiguous source with a dtype conversion: would previously sync.
|
||||
src = torch.arange(12, dtype=torch.int64).reshape(3, 4).T
|
||||
result = async_tensor_h2d(src, device=device, dtype=torch.int32)
|
||||
assert result.dtype == torch.int32
|
||||
assert result.is_contiguous()
|
||||
assert torch.equal(result.cpu(), src.to(torch.int32))
|
||||
|
||||
# numpy and list sources still work.
|
||||
assert torch.equal(
|
||||
async_tensor_h2d(np.arange(4, dtype=np.int64), device=device),
|
||||
torch.arange(4, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
async_tensor_h2d([1, 2, 3], device=device, dtype=torch.int32),
|
||||
torch.tensor([1, 2, 3], dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
@@ -996,7 +996,7 @@ def test_pcp_plan_deinterleave_restores_global_order(
|
||||
|
||||
monkeypatch.setattr("vllm.v1.attention.backends.mla.indexer.PIN_MEMORY", False)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.indexer.async_copy_to_gpu",
|
||||
"vllm.v1.attention.backends.mla.indexer.async_tensor_h2d",
|
||||
lambda x, device: x.to(device),
|
||||
)
|
||||
scheduled = np.array(req_lens, dtype=np.int64)
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers, set_dummy_c
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
|
||||
def _copy_to_cpu(value, out=None, device=None):
|
||||
def _copy_to_cpu(value, device=None, dtype=None, out=None):
|
||||
tensor = torch.from_numpy(value) if isinstance(value, np.ndarray) else value
|
||||
if out is not None:
|
||||
return out.copy_(tensor)
|
||||
@@ -83,7 +83,7 @@ def test_replicated_decode_piecewise_graph_padding(monkeypatch):
|
||||
device=torch.device("cpu"),
|
||||
dcp_world_size=1,
|
||||
)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
|
||||
|
||||
segments_by_rank, per_rank_num_tokens = manager._build_batch_layout(
|
||||
num_scheduled_tokens=np.ones(3, dtype=np.int32),
|
||||
@@ -158,7 +158,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
|
||||
device=torch.device("cpu"),
|
||||
dcp_world_size=1,
|
||||
)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
|
||||
|
||||
with pytest.raises(ValueError, match="smaller than the largest rank-local batch"):
|
||||
manager._build_batch_layout(
|
||||
|
||||
@@ -269,11 +269,11 @@ from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
async_tensor_h2d,
|
||||
direct_register_custom_op,
|
||||
get_dtype_size,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
np_to_pinned_tensor,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
@@ -1980,11 +1980,6 @@ def plan_mla_context_chunks(
|
||||
return plans
|
||||
|
||||
|
||||
def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor:
|
||||
"""Pinned int32 CPU tensor backing one concatenated per-chunk field."""
|
||||
return np_to_pinned_tensor(np.asarray(values, dtype=np.int32))
|
||||
|
||||
|
||||
def align_mla_chunked_context_workspace_size(
|
||||
vllm_config: VllmConfig,
|
||||
workspace_size: int,
|
||||
@@ -2161,24 +2156,28 @@ def build_mla_chunked_context_metadata(
|
||||
token_offset += num_tokens
|
||||
local_token_offset += num_local_tokens
|
||||
|
||||
seq_lens_cpu = _flat_int32(seq_lens_flat)
|
||||
starts_and_context_lens = _flat_int32(starts_flat + context_lens).to(
|
||||
device, non_blocking=True
|
||||
seq_lens_cpu = torch.tensor(
|
||||
seq_lens_flat, dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
starts_and_context_lens = async_tensor_h2d(
|
||||
starts_flat + context_lens, device=device, dtype=torch.int32
|
||||
)
|
||||
starts = starts_and_context_lens[: len(starts_flat)]
|
||||
context_lens_gpu = starts_and_context_lens[len(starts_flat) :]
|
||||
cu_seq_lens = _flat_int32(cu_seq_lens_flat).to(device, non_blocking=True)
|
||||
cu_seqlens_q = _flat_int32(cu_seqlens_q_flat).to(device, non_blocking=True)
|
||||
token_to_seq = _flat_int32(np.concatenate(token_to_seq_parts)).to(
|
||||
device, non_blocking=True
|
||||
cu_seq_lens = async_tensor_h2d(cu_seq_lens_flat, device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = async_tensor_h2d(cu_seqlens_q_flat, device=device, dtype=torch.int32)
|
||||
token_to_seq = async_tensor_h2d(
|
||||
np.concatenate(token_to_seq_parts), device=device, dtype=torch.int32
|
||||
)
|
||||
if use_dcp:
|
||||
padded_local_cu_seq_lens = _flat_int32(padded_local_cu_seq_lens_flat).to(
|
||||
device, non_blocking=True
|
||||
padded_local_cu_seq_lens = async_tensor_h2d(
|
||||
padded_local_cu_seq_lens_flat, device=device, dtype=torch.int32
|
||||
)
|
||||
padded_local_token_to_seq = async_tensor_h2d(
|
||||
np.concatenate(padded_local_token_to_seq_parts),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
padded_local_token_to_seq = _flat_int32(
|
||||
np.concatenate(padded_local_token_to_seq_parts)
|
||||
).to(device, non_blocking=True)
|
||||
|
||||
chunks: list[MLACommonPrefillMetadata.ContextChunk] = []
|
||||
for index, (plan, layout) in enumerate(zip(plans, layouts)):
|
||||
|
||||
@@ -53,7 +53,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor, async_copy_to_gpu
|
||||
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
|
||||
@@ -1159,7 +1159,7 @@ class DiffusionSampler:
|
||||
return
|
||||
# Move the slot indices across once, up front: indexing a device
|
||||
# tensor with a numpy array copies them over synchronously each time.
|
||||
ps_gpu = async_copy_to_gpu(
|
||||
ps_gpu = async_tensor_h2d(
|
||||
ps.astype(np.int64), device=states.is_encoder_phase.device
|
||||
)
|
||||
states.init_canvas(ps_gpu)
|
||||
@@ -1266,7 +1266,7 @@ class DiffusionSampler:
|
||||
# was truncated near max_model_len, in which case the scheduler gave us
|
||||
# fewer than CL logits for that request.
|
||||
valid_canvas_len_np = per_req_nlogits_np[per_req_nlogits_np > 0]
|
||||
valid_canvas_len = async_copy_to_gpu(
|
||||
valid_canvas_len = async_tensor_h2d(
|
||||
valid_canvas_len_np.astype(np.int64), device=device
|
||||
)
|
||||
|
||||
|
||||
@@ -19,7 +19,11 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
async_tensor_h2d,
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
AttentionLayer,
|
||||
@@ -231,8 +235,12 @@ def _build_sliding_window_metadata(
|
||||
query_lens = query_lens_cpu[req_start:req_end]
|
||||
kv_lens = kv_lens_cpu[req_start:req_end]
|
||||
num_reqs = req_end - req_start
|
||||
cu_seq_lens_q_cpu = torch.zeros(num_reqs + 1, dtype=torch.int32)
|
||||
cu_seq_lens_k_cpu = torch.zeros(num_reqs + 1, dtype=torch.int32)
|
||||
cu_seq_lens_q_cpu = torch.zeros(
|
||||
num_reqs + 1, dtype=torch.int32, pin_memory=PIN_MEMORY
|
||||
)
|
||||
cu_seq_lens_k_cpu = torch.zeros(
|
||||
num_reqs + 1, dtype=torch.int32, pin_memory=PIN_MEMORY
|
||||
)
|
||||
torch.cumsum(query_lens, 0, out=cu_seq_lens_q_cpu[1:])
|
||||
torch.cumsum(kv_lens, 0, out=cu_seq_lens_k_cpu[1:])
|
||||
token_to_seq_cpu = torch.repeat_interleave(
|
||||
@@ -248,8 +256,8 @@ def _build_sliding_window_metadata(
|
||||
query_end=query_end,
|
||||
cu_seq_lens_q=cu_seq_lens_q_cpu.to(device, non_blocking=True),
|
||||
cu_seq_lens_k=cu_seq_lens_k_cpu.to(device, non_blocking=True),
|
||||
starts=starts_cpu[req_start:req_end].to(device, non_blocking=True),
|
||||
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
|
||||
starts=async_tensor_h2d(starts_cpu[req_start:req_end], device),
|
||||
token_to_seq=async_tensor_h2d(token_to_seq_cpu, device),
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
max_seq_len_q=int(query_lens.max().item()),
|
||||
max_seq_len_k=int(kv_lens.max().item()),
|
||||
@@ -396,8 +404,8 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder):
|
||||
seq_lens = common_attn_metadata.seq_lens[
|
||||
reqs_start + req_start : reqs_start + req_end
|
||||
]
|
||||
query_lens = query_lens_cpu[req_start:req_end].to(
|
||||
self.device, non_blocking=True
|
||||
query_lens = async_tensor_h2d(
|
||||
query_lens_cpu[req_start:req_end], self.device
|
||||
)
|
||||
kv_lens = torch.minimum(seq_lens, query_lens + self.sliding_window - 1)
|
||||
chunk.cu_seq_lens_k.zero_()
|
||||
|
||||
+3
-2
@@ -10,6 +10,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
from .utils import tensor_cache
|
||||
@@ -27,8 +28,8 @@ def prepare_chunk_indices(cu_seqlens: torch.Tensor, chunk_size: int) -> torch.Te
|
||||
chunk_counts = triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist()
|
||||
indices = torch.cat([torch.arange(n) for n in chunk_counts])
|
||||
chunk_indices = torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1)
|
||||
return chunk_indices.to(
|
||||
device=cu_seqlens.device, dtype=cu_seqlens.dtype, non_blocking=True
|
||||
return async_tensor_h2d(
|
||||
chunk_indices, device=cu_seqlens.device, dtype=cu_seqlens.dtype
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -705,17 +705,30 @@ def create_kv_caches_with_random(
|
||||
|
||||
def async_tensor_h2d(
|
||||
data: list | np.ndarray | torch.Tensor,
|
||||
device: str | torch.device,
|
||||
device: str | torch.device | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Copy list/numpy array/tensor async from host to device."""
|
||||
if dtype is None and out is not None:
|
||||
dtype = out.dtype
|
||||
if isinstance(data, np.ndarray):
|
||||
data = torch.from_numpy(data)
|
||||
if isinstance(data, torch.Tensor):
|
||||
t = data.pin_memory() if PIN_MEMORY else data
|
||||
t = data
|
||||
if PIN_MEMORY and not t.is_pinned():
|
||||
# Stage in pinned, contiguous buffer to ensure fully async copy.
|
||||
t = torch.empty(
|
||||
t.shape, dtype=dtype or t.dtype, device="cpu", pin_memory=True
|
||||
).copy_(t)
|
||||
else:
|
||||
t = torch.tensor(data, dtype=dtype, pin_memory=PIN_MEMORY, device="cpu")
|
||||
assert t.is_cpu
|
||||
|
||||
if out is not None:
|
||||
assert out.dtype == dtype
|
||||
return out.copy_(t, non_blocking=True)
|
||||
assert device is not None, "must provide destination tensor or device"
|
||||
return t.to(device=device, dtype=dtype, non_blocking=True)
|
||||
|
||||
|
||||
|
||||
@@ -1264,15 +1264,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
"""
|
||||
# write self.paged_kv_indptr_cpu inplace (0-index is always 0)
|
||||
np.cumsum(
|
||||
num_blocks_np,
|
||||
dtype=np.int32,
|
||||
out=self.paged_kv_indptr.np[1 : num_reqs + 1],
|
||||
num_blocks_np, dtype=np.int32, out=self.paged_kv_indptr.np[1 : num_reqs + 1]
|
||||
)
|
||||
paged_kv_indptr = self.paged_kv_indptr.gpu[: num_reqs + 1]
|
||||
paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_reqs + 1]
|
||||
if PIN_MEMORY:
|
||||
paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory()
|
||||
paged_kv_indptr.copy_(paged_kv_indptr_cpu, non_blocking=True)
|
||||
paged_kv_indptr = self.paged_kv_indptr.copy_to_gpu(num_reqs + 1)
|
||||
|
||||
# write self.paged_kv_indices inplace
|
||||
num_actual_pages = self.paged_kv_indptr.np[num_reqs]
|
||||
@@ -1292,12 +1286,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
page_size,
|
||||
paged_kv_last_page_len_np,
|
||||
)
|
||||
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[:num_reqs]
|
||||
if PIN_MEMORY:
|
||||
paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory()
|
||||
self.paged_kv_last_page_len.gpu[:num_reqs].copy_(
|
||||
paged_kv_last_page_len_cpu, non_blocking=True
|
||||
)
|
||||
self.paged_kv_last_page_len.copy_to_gpu(num_reqs)
|
||||
return paged_kv_indices
|
||||
|
||||
def build(
|
||||
|
||||
@@ -20,7 +20,11 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import current_stream, is_quantized_kv_cache
|
||||
from vllm.utils.torch_utils import (
|
||||
async_tensor_h2d,
|
||||
current_stream,
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -48,7 +52,6 @@ from vllm.v1.attention.ops.flashmla import (
|
||||
get_mla_metadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -519,7 +522,7 @@ class FlashMLASparseMetadataBuilder(
|
||||
)
|
||||
workspace_rows = torch.from_numpy(rows_per_rank.astype(np.int32))
|
||||
max_prefill_buffer_size //= self.dcp_world_size
|
||||
entry_rows = async_copy_to_gpu(row_bounds[:-1], device=self.device)
|
||||
entry_rows = async_tensor_h2d(row_bounds[:-1], device=self.device)
|
||||
prefill_block_table = prefill_block_table.index_select(0, entry_rows)
|
||||
prefill_seq_lens = prefill_seq_lens.index_select(0, entry_rows)
|
||||
num_entries = len(workspace_rows)
|
||||
|
||||
@@ -27,7 +27,7 @@ from vllm.utils.deep_gemm import (
|
||||
)
|
||||
from vllm.utils.math_utils import round_down
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -47,7 +47,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -338,14 +337,14 @@ def build_pcp_global_chunk_plan(
|
||||
+ local
|
||||
)
|
||||
|
||||
cu = async_copy_to_gpu(cu, device=device)
|
||||
cu = async_tensor_h2d(cu, device=device)
|
||||
return PCPGlobalChunkPlan(
|
||||
row_start_cu=cu[0],
|
||||
global_cu=cu[1],
|
||||
padded_local_cu=cu[2],
|
||||
padded_local_total=padded_total,
|
||||
total=total,
|
||||
deinterleave_idx=async_copy_to_gpu(idx, device=device),
|
||||
deinterleave_idx=async_tensor_h2d(idx, device=device),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
|
||||
from vllm.v1.sample.logits_processor.builtin import MinTokensLogitsProcessor
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
@@ -327,9 +328,7 @@ class RejectionSampler(nn.Module):
|
||||
num_draft_tokens = torch.tensor(metadata.num_draft_tokens, device="cpu")
|
||||
original_indices = torch.arange(num_requests, device="cpu")
|
||||
repeat_indices_cpu = original_indices.repeat_interleave(num_draft_tokens)
|
||||
repeat_indices = repeat_indices_cpu.to(
|
||||
device=logits.device, non_blocking=True
|
||||
)
|
||||
repeat_indices = async_tensor_h2d(repeat_indices_cpu, logits.device)
|
||||
logits = self.apply_penalties(
|
||||
logits, sampling_metadata, metadata, repeat_indices, output_token_ids
|
||||
)
|
||||
|
||||
@@ -255,7 +255,6 @@ class SpecDecodeBaseProposer:
|
||||
self.backup_next_token_ids = CpuGpuBuffer(
|
||||
self.max_batch_size,
|
||||
dtype=torch.int32,
|
||||
pin_memory=PIN_MEMORY,
|
||||
device=device,
|
||||
with_numpy=True,
|
||||
)
|
||||
|
||||
+4
-3
@@ -137,9 +137,10 @@ class CpuGpuBuffer:
|
||||
self.np = self.cpu.numpy()
|
||||
|
||||
def copy_to_gpu(self, n: int | None = None) -> torch.Tensor:
|
||||
if n is None:
|
||||
return self.gpu.copy_(self.cpu, non_blocking=True)
|
||||
return self.gpu[:n].copy_(self.cpu[:n], non_blocking=True)
|
||||
cpu, gpu = self.cpu, self.gpu
|
||||
if n is not None:
|
||||
cpu, gpu = cpu[:n], gpu[:n]
|
||||
return gpu.copy_(cpu.pin_memory() if PIN_MEMORY else cpu, non_blocking=True)
|
||||
|
||||
def copy_to_cpu(self, n: int | None = None) -> torch.Tensor:
|
||||
"""NOTE: Because this method is non-blocking, explicit synchronization
|
||||
|
||||
@@ -76,14 +76,17 @@ import vllm.utils.torch_utils as torch_utils
|
||||
|
||||
def async_tensor_h2d(
|
||||
data: list | np.ndarray | torch.Tensor,
|
||||
device: str | torch.device,
|
||||
device: str | torch.device | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if isinstance(data, np.ndarray):
|
||||
data = torch.from_numpy(data)
|
||||
if isinstance(data, torch.Tensor):
|
||||
if not isinstance(data, torch.Tensor):
|
||||
data = torch.tensor(data, dtype=dtype, device="cpu")
|
||||
elif out is None:
|
||||
return data.to(dtype=dtype)
|
||||
return torch.tensor(data, dtype=dtype, device="cpu")
|
||||
return data if out is None else out.copy_(data)
|
||||
|
||||
|
||||
torch_utils.async_tensor_h2d = async_tensor_h2d
|
||||
|
||||
@@ -10,7 +10,6 @@ from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
async_tensor_h2d,
|
||||
get_accelerator_view_from_cpu_tensor,
|
||||
)
|
||||
@@ -27,24 +26,6 @@ def set_default_max_concurrency(n: int) -> None:
|
||||
_DEFAULT_MAX_CONCURRENCY = max(2, n)
|
||||
|
||||
|
||||
def async_copy_to_gpu(
|
||||
x: torch.Tensor | np.ndarray,
|
||||
out: torch.Tensor | None = None,
|
||||
device: torch.device | None = None,
|
||||
) -> torch.Tensor:
|
||||
if isinstance(x, np.ndarray):
|
||||
x = torch.from_numpy(x)
|
||||
assert x.is_cpu
|
||||
|
||||
if out is None:
|
||||
assert device is not None
|
||||
out = torch.empty_like(x, device=device)
|
||||
|
||||
# pin_memory() is no-op if the memory is already pinned.
|
||||
pinned = x.pin_memory() if PIN_MEMORY else x
|
||||
return out.copy_(pinned, non_blocking=True)
|
||||
|
||||
|
||||
class UvaBuffer:
|
||||
def __init__(self, size: int | Sequence[int], dtype: torch.dtype):
|
||||
if not is_uva_available():
|
||||
|
||||
@@ -64,7 +64,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, async_tensor_h2d
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
@@ -99,10 +99,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
init_kv_cache,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.buffer_utils import (
|
||||
async_copy_to_gpu,
|
||||
set_default_max_concurrency,
|
||||
)
|
||||
from vllm.v1.worker.gpu.buffer_utils import set_default_max_concurrency
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
@@ -1289,7 +1286,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_ids = batch_req_state.req_ids
|
||||
num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens
|
||||
idx_mapping_np = batch_req_state.idx_mapping_np
|
||||
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
|
||||
idx_mapping = async_tensor_h2d(idx_mapping_np, device=self.device)
|
||||
num_reqs = len(req_ids)
|
||||
|
||||
# Get the number of draft tokens for each request.
|
||||
@@ -1324,7 +1321,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32)
|
||||
cu_num_logits_np[0] = 0
|
||||
np.cumsum(num_logits, out=cu_num_logits_np[1:])
|
||||
cu_num_logits = async_copy_to_gpu(cu_num_logits_np, device=self.device)
|
||||
cu_num_logits = async_tensor_h2d(cu_num_logits_np, device=self.device)
|
||||
|
||||
adaptive_verification = (
|
||||
self.adaptive_verification if num_draft_tokens_per_req is not None else None
|
||||
@@ -1352,7 +1349,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Some attention backends like FA3 require query_start_loc to be non-decreasing.
|
||||
query_start_loc_np[num_reqs + 1 :] = num_tokens
|
||||
query_start_loc = self.input_buffers.query_start_loc
|
||||
async_copy_to_gpu(query_start_loc_np, out=query_start_loc)
|
||||
async_tensor_h2d(query_start_loc_np, out=query_start_loc)
|
||||
if adaptive_verification is not None:
|
||||
cu_num_logits, query_start_loc, total_num_draft_tokens = (
|
||||
adaptive_verification.reallocate_drafts(req_ids, idx_mapping)
|
||||
|
||||
@@ -10,9 +10,9 @@ import torch
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.parallel_state import get_dcp_group, get_pcp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID, get_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.v1.worker.gpu.input_batch import (
|
||||
InputBatch,
|
||||
InputBuffers,
|
||||
@@ -358,13 +358,13 @@ class PCPManager:
|
||||
dtype=np.int64,
|
||||
)
|
||||
|
||||
self._hidden_restore_idx = async_copy_to_gpu(
|
||||
self._hidden_restore_idx = async_tensor_h2d(
|
||||
hidden_restore_idx, device=self.device
|
||||
)
|
||||
self._padded_gather_idx = async_copy_to_gpu(
|
||||
self._padded_gather_idx = async_tensor_h2d(
|
||||
padded_gather_idx, device=self.device
|
||||
)
|
||||
self._gathered_kv_write_mask = async_copy_to_gpu(
|
||||
self._gathered_kv_write_mask = async_tensor_h2d(
|
||||
gathered_kv_write_mask, device=self.device
|
||||
)
|
||||
return segments_by_rank, per_rank_num_tokens
|
||||
@@ -536,12 +536,12 @@ class PCPManager:
|
||||
local_query_start_loc_out = local_query_start_loc_np[1 : num_local_reqs + 1]
|
||||
np.cumsum(local_num_scheduled_tokens, out=local_query_start_loc_out)
|
||||
local_query_start_loc_np[num_local_reqs + 1 :] = num_local_tokens
|
||||
async_copy_to_gpu(local_query_start_loc_np, out=input_buffers.query_start_loc)
|
||||
async_tensor_h2d(local_query_start_loc_np, out=input_buffers.query_start_loc)
|
||||
local_query_start_loc = input_buffers.query_start_loc[
|
||||
: num_reqs_after_padding + 1
|
||||
]
|
||||
|
||||
local_to_global_req_idx = async_copy_to_gpu(
|
||||
local_to_global_req_idx = async_tensor_h2d(
|
||||
local_to_global_req_idx_np, device=self.device
|
||||
)
|
||||
seq_lens = input_buffers.seq_lens[:num_reqs_after_padding]
|
||||
|
||||
@@ -11,7 +11,7 @@ import torch
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
|
||||
@@ -138,7 +138,7 @@ class PPHandler:
|
||||
return None
|
||||
# Filter excluded request indices.
|
||||
idx_mapping_np = np.where(exclude_mask, -1, slot.idx_mapping_np)
|
||||
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
|
||||
idx_mapping = async_tensor_h2d(idx_mapping_np, device=self.device)
|
||||
|
||||
self.main_stream.wait_event(slot.event)
|
||||
if slot.draft_tokens is not None and draft_tokens_to_update is not None:
|
||||
@@ -148,7 +148,7 @@ class PPHandler:
|
||||
keep = ~exclude_mask
|
||||
keep_t = torch.as_tensor(keep, device=self.device)
|
||||
draft_tokens = draft_tokens[keep_t]
|
||||
draft_idx_mapping = async_copy_to_gpu(
|
||||
draft_idx_mapping = async_tensor_h2d(
|
||||
slot.idx_mapping_np[keep], device=self.device
|
||||
)
|
||||
draft_tokens_to_update[draft_idx_mapping] = draft_tokens
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.utils import CpuGpuBuffer
|
||||
from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream
|
||||
@@ -21,7 +22,6 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
get_attn_cg_support,
|
||||
get_query_lens_mismatch_unsupported_backend,
|
||||
)
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_PROFILE_REPLAYS = 5
|
||||
@@ -413,7 +413,7 @@ class AdaptiveVerificationManager:
|
||||
if draft_budget == 0:
|
||||
capacities.zero_()
|
||||
else:
|
||||
async_copy_to_gpu(scheduled_drafts, out=capacities)
|
||||
async_tensor_h2d(scheduled_drafts, out=capacities)
|
||||
if draft_budget < int(scheduled_drafts.sum()):
|
||||
_assign_draft_token_budget_compiled(
|
||||
self._confidence_probs,
|
||||
@@ -424,10 +424,7 @@ class AdaptiveVerificationManager:
|
||||
)
|
||||
|
||||
num_non_draft_tokens_gpu = self._num_non_draft_tokens[:num_reqs]
|
||||
async_copy_to_gpu(
|
||||
num_non_draft_tokens,
|
||||
out=num_non_draft_tokens_gpu,
|
||||
)
|
||||
async_tensor_h2d(num_non_draft_tokens, out=num_non_draft_tokens_gpu)
|
||||
self._cu_num_logits[:1].zero_()
|
||||
torch.cumsum(
|
||||
capacities + self.num_bonus_tokens,
|
||||
|
||||
@@ -5,8 +5,7 @@ import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
|
||||
@@ -68,7 +67,7 @@ class StructuredOutputsWorker:
|
||||
|
||||
# Asynchronously copy the bitmask to GPU.
|
||||
with torch.cuda.stream(self.copy_stream):
|
||||
bitmask = async_copy_to_gpu(
|
||||
bitmask = async_tensor_h2d(
|
||||
grammar_bitmask, out=self.grammar_bitmask[: grammar_bitmask.shape[0]]
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user