[Core] Ensure memory is pinned prior to async h2d copy (#45424)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-20 20:02:24 -07:00
committed by GitHub
parent 8dd1b702f2
commit 7df3d7dada
49 changed files with 254 additions and 264 deletions
@@ -145,7 +145,6 @@ def _generate_fake_sampling_metadata(
vllm_config.scheduler_config.max_num_seqs,
num_spec,
device,
PIN_MEMORY_AVAILABLE,
)
fake_sampling_metadata = SamplingMetadata(
temperature=torch.full((batch_size,), 0.0),
@@ -880,7 +879,6 @@ def test_maybe_create_thinking_budget_holder_without_reasoning():
cfg.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
False,
)
is None
)
@@ -35,7 +35,6 @@ def mock_model_runner_with_input_batch():
max_model_len=1024,
max_num_batched_tokens=1024,
device="cpu",
pin_memory=False,
vocab_size=32000,
block_sizes=[16],
kernel_block_sizes=[16],
-6
View File
@@ -10,7 +10,6 @@ import torch
from vllm.platforms import current_platform
from vllm.sampling_params import SamplingParams
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import make_tensor_with_pad
from vllm.v1.pool.metadata import PoolingMetadata
from vllm.v1.sample.logits_processor import LogitsProcessors
@@ -236,7 +235,6 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
max_model_len=1024,
max_num_batched_tokens=1024,
device=torch.device(device),
pin_memory=is_pin_memory_available(),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
@@ -331,7 +329,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
max_model_len=1024,
max_num_batched_tokens=1024,
device=torch.device(device),
pin_memory=is_pin_memory_available(),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
@@ -341,7 +338,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
max_model_len=1024,
max_num_batched_tokens=1024,
device=torch.device(device),
pin_memory=is_pin_memory_available(),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
@@ -410,7 +406,6 @@ def test_pooling_prompt_lens_not_aliased(device: str):
max_model_len=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS,
max_num_batched_tokens=batch_size * (MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS),
device=torch.device(device),
pin_memory=is_pin_memory_available(),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
@@ -459,7 +454,6 @@ def test_pooling_metadata_token_id_buffers(
max_model_len=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS,
max_num_batched_tokens=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS,
device=torch.device("cpu"),
pin_memory=False,
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
-4
View File
@@ -85,7 +85,6 @@ def initialize_kv_cache(runner: GPUModelRunner):
max_model_len=runner.max_model_len,
max_num_batched_tokens=runner.max_num_tokens,
device=runner.device,
pin_memory=runner.pin_memory,
vocab_size=runner.model_config.get_vocab_size(),
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
kernel_block_sizes=[
@@ -1405,7 +1404,6 @@ def test_input_batch_with_kernel_block_sizes():
max_model_len = 512
max_num_batched_tokens = 512
device = torch.device(DEVICE_TYPE)
pin_memory = False
vocab_size = 50272
# Test with different kernel block sizes
@@ -1417,7 +1415,6 @@ def test_input_batch_with_kernel_block_sizes():
max_model_len=max_model_len,
max_num_batched_tokens=max_num_batched_tokens,
device=device,
pin_memory=pin_memory,
vocab_size=vocab_size,
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
@@ -1478,7 +1475,6 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
max_model_len=runner.max_model_len,
max_num_batched_tokens=runner.max_num_tokens,
device=runner.device,
pin_memory=runner.pin_memory,
vocab_size=runner.model_config.get_vocab_size(),
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
kernel_block_sizes=[16],
+2 -2
View File
@@ -18,8 +18,8 @@ import torch
from vllm.device_allocator import AllocationData, HandleType
from vllm.logger import init_logger
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.system_utils import find_loaded_library
from vllm.utils.torch_utils import PIN_MEMORY
logger = init_logger(__name__)
@@ -196,7 +196,7 @@ class CuMemAllocator:
size_in_bytes,
dtype=torch.uint8,
device="cpu",
pin_memory=is_pin_memory_available(),
pin_memory=PIN_MEMORY,
)
cpu_ptr = cpu_backup_tensor.data_ptr()
libcudart.cudaMemcpy(cpu_ptr, ptr, size_in_bytes)
+2 -2
View File
@@ -11,7 +11,7 @@ import torch
from vllm.device_allocator import AllocationData, HandleType
from vllm.logger import init_logger
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
logger = init_logger(__name__)
@@ -188,7 +188,7 @@ class XpuMemAllocator:
size_in_bytes,
dtype=torch.uint8,
device="cpu",
pin_memory=is_pin_memory_available(),
pin_memory=PIN_MEMORY,
)
cpu_ptr = cpu_backup_tensor.data_ptr()
_xpu_memcpy_sync(
+2 -2
View File
@@ -17,7 +17,7 @@ from vllm.lora.utils import (
)
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
from vllm.model_executor.models.utils import WeightsMapper
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
logger = init_logger(__name__)
@@ -126,7 +126,7 @@ class LoRAModel:
skip_prefixes: list[str] | None = None,
) -> "LoRAModel":
"""Create a LoRAModel from a dictionary of tensors."""
pin_memory = str(device) == "cpu" and is_pin_memory_available()
pin_memory = str(device) == "cpu" and PIN_MEMORY
loras: dict[str, LoRALayerWeights] = {}
for tensor_name, tensor in tensors.items():
if is_base_embedding_weights(tensor_name):
+2 -2
View File
@@ -7,7 +7,7 @@ import torch
import torch.types
from vllm.lora.peft_helper import PEFTHelper
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
class LoRALayerWeights:
@@ -79,7 +79,7 @@ class LoRALayerWeights:
dtype: torch.dtype,
device: torch.types.Device,
) -> "LoRALayerWeights":
pin_memory = str(device) == "cpu" and is_pin_memory_available()
pin_memory = str(device) == "cpu" and PIN_MEMORY
lora_a = torch.zeros(
[rank, input_dim], dtype=dtype, device=device, pin_memory=pin_memory
)
+2 -2
View File
@@ -42,7 +42,7 @@ from vllm.model_executor.models.utils import PPMissingLayer
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.encoder_budget import MultiModalBudget
from vllm.utils.cache import LRUCache
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
logger = init_logger(__name__)
@@ -801,7 +801,7 @@ class LoRAModelManager:
# 2. The weight packing above (e.g., pack_moe) may invalidate the
# pin_memory allocation, so we execute it after packing.
pin_memory = str(lora_device) == "cpu" and is_pin_memory_available()
pin_memory = str(lora_device) == "cpu" and PIN_MEMORY
if pin_memory:
for lora in lora_model.loras.values():
if isinstance(lora.lora_a, list):
@@ -1684,12 +1684,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
# [[0, 0, 0, 0], [256, 256, 256, 256], [512, 512, 512, 512]]
# Note(simon): this is done in CPU because of downstream's
# of `to_list`.
chunk_starts = (
chunk_starts = torch.empty(
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
).copy_(
torch.arange(num_chunks, dtype=torch.int32)
.multiply_(max_context_chunk)
.unsqueeze(1)
.expand(-1, num_prefills)
* max_context_chunk
).pin_memory()
)
chunk_ends = torch.min(
context_lens_cpu.unsqueeze(0), chunk_starts + max_context_chunk
)
@@ -1746,12 +1747,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
)
* self.dcp_local_block_size
)
local_chunk_starts = (
local_chunk_starts = torch.empty(
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
).copy_(
torch.arange(num_chunks, dtype=torch.int32)
.multiply_(padded_local_max_context_chunk_across_ranks)
.unsqueeze(1)
.expand(-1, num_prefills)
* padded_local_max_context_chunk_across_ranks
).pin_memory()
)
local_chunk_ends = torch.min(
padded_local_context_lens_cpu.unsqueeze(0),
local_chunk_starts
@@ -28,6 +28,7 @@ from vllm.utils.flashinfer import (
is_flashinfer_cudnn_fp8_prefill_attn_supported,
)
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.ops.vit_attn_wrappers import (
@@ -311,7 +312,7 @@ class MMEncoderAttention(CustomOp):
)
cu_seqlens = np.concatenate([cu_seqlens_qko, cu_seqlens_v])
cu_seqlens = torch.from_numpy(cu_seqlens).to(device, non_blocking=True)
cu_seqlens = async_tensor_h2d(cu_seqlens, device=device)
return cu_seqlens
def __init__(
@@ -10,6 +10,7 @@ import torch.nn as nn
from vllm.config.pooler import SequencePoolingType
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.pool.metadata import PoolingMetadata
SequencePoolingMethodOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
@@ -74,15 +75,14 @@ class MeanPool(SequencePoolingMethod):
# early return for empty batch
return hidden_states.new_empty((0, hidden_size), dtype=torch.float32)
# Build segment_ids on CPU so repeat_interleave doesn't need to sync
# GPU->CPU to learn its data-dependent output length, then upload
# non-blocking. eg. [2, 1, 3] -> [0, 0, 1, 2, 2, 2]
prompt_lens = async_tensor_h2d(
prompt_lens_cpu, device=hidden_states.device, dtype=torch.int64
)
# eg. [2, 1, 3] -> [0, 0, 1, 2, 2, 2]
segment_ids = torch.repeat_interleave(
torch.arange(num_seqs, dtype=torch.long),
prompt_lens_cpu,
).to(hidden_states.device, non_blocking=True)
prompt_lens = prompt_lens_cpu.to(
hidden_states.device, dtype=torch.int64, non_blocking=True
torch.arange(num_seqs, device=hidden_states.device, dtype=torch.long),
prompt_lens,
output_size=int(prompt_lens_cpu.sum()),
)
segment_sums = torch.zeros(
(num_seqs, hidden_size),
+3 -2
View File
@@ -66,6 +66,7 @@ from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.vision import is_vit_use_data_parallel
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.moonvit import MoonViTConfig
from vllm.utils.torch_utils import async_tensor_h2d
def _apply_rope_input_validation(x, freqs_cis):
@@ -758,7 +759,7 @@ class MoonVitPretrainedModel(PreTrainedModel):
),
]
)
metadata["cu_seqlens"] = torch.from_numpy(cu_seqlens_np).to(device)
metadata["cu_seqlens"] = async_tensor_h2d(cu_seqlens_np, device=device)
if max_seqlen_override is not None:
max_seqlen_val = int(max_seqlen_override)
@@ -770,7 +771,7 @@ class MoonVitPretrainedModel(PreTrainedModel):
metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)
gather_idx_np = _build_merge_gather_idx(grid_pairs, self.merge_kernel_size)
metadata["merge_gather_idx"] = torch.from_numpy(gather_idx_np).to(device)
metadata["merge_gather_idx"] = async_tensor_h2d(gather_idx_np, device=device)
return metadata
+2 -3
View File
@@ -83,9 +83,8 @@ from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
@@ -825,7 +824,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
@staticmethod
def invert_permutation(perm: torch.Tensor) -> torch.Tensor:
# building the inverse permutation in O(n) time
inv = torch.empty_like(perm, pin_memory=is_pin_memory_available())
inv = torch.empty_like(perm, pin_memory=PIN_MEMORY)
inv[perm] = torch.arange(perm.numel(), device=perm.device, dtype=perm.dtype)
return inv
+2 -1
View File
@@ -13,6 +13,7 @@ from vllm.config.cache import CacheDType
from vllm.platforms.interface import DeviceCapability
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import np_to_pinned_tensor
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -207,7 +208,7 @@ class DeepseekV4FlashMLAMetadataBuilder(
# Zero-fill for cudagraphs
self.req_id_per_token_buffer.fill_(0)
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
torch.from_numpy(req_id_per_token), non_blocking=True
np_to_pinned_tensor(req_id_per_token), non_blocking=True
)
req_id_per_token = self.req_id_per_token_buffer[:num_tokens]
+14 -2
View File
@@ -488,7 +488,13 @@ class MultiModalBatchedField(BaseMultiModalField):
# An optimization when `batch` contains only one tensor:
# - produce exactly same result as `torch.stack(batch)`
# - will achieve zero-copy if the tensor is contiguous
return batch[0].unsqueeze(0).contiguous()
out = batch[0].unsqueeze(0)
if not pin_memory:
return out.contiguous()
# Avoid extra copy - pinning unpinned memory will make it contiguous
if not out.is_contiguous() and out.is_pinned():
out = out.contiguous()
return out.pin_memory()
first_shape = batch[0].shape
if all(elem.shape == first_shape for elem in batch):
out = torch.empty(
@@ -538,7 +544,13 @@ class MultiModalFlatField(BaseMultiModalField):
# An optimization when `batch` contains only one tensor:
# - produce exactly same result as `torch.concat(batch)`
# - will achieve zero-copy if the tensor is contiguous
return batch[0].contiguous()
out = batch[0]
if not pin_memory:
return out.contiguous()
# Avoid extra copy - pinning unpinned memory will make it contiguous
if not out.is_contiguous() and out.is_pinned():
out = out.contiguous()
return out.pin_memory()
dim = self.dim + (self.dim < 0) * len(batch[0].shape)
+1 -2
View File
@@ -9,7 +9,6 @@ from typing import TYPE_CHECKING
from vllm import envs
from vllm.plugins import PLATFORM_PLUGINS_GROUP, load_plugins_by_group
from vllm.utils.import_utils import resolve_obj_by_qualname
from vllm.utils.torch_utils import supports_xccl
from .interface import CpuArchEnum, Platform, PlatformEnum
@@ -135,7 +134,7 @@ def xpu_platform_plugin() -> str | None:
try:
import torch
if supports_xccl():
if torch.distributed.is_xccl_available():
dist_backend = "xccl"
from vllm.platforms.xpu import XPUPlatform
+2 -1
View File
@@ -23,7 +23,6 @@ import vllm._C_stable_libtorch # noqa
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.utils.import_utils import import_pynvml
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interface import DeviceCapability, Platform, PlatformEnum, in_wsl
@@ -88,6 +87,8 @@ def _get_backend_priorities(
kv_cache_dtype: CacheDType | None = None,
) -> list[AttentionBackendEnum]:
"""Get backend priorities with lazy import to avoid circular dependency."""
from vllm.utils.torch_utils import is_quantized_kv_cache
if use_mla:
if device_capability.major == 10:
# Sparse MLA backend priorities
+6 -6
View File
@@ -14,7 +14,6 @@ import vllm_xpu_kernels._xpu_C # noqa
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.utils.torch_utils import supports_xpu_graph
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interface import DeviceCapability, Platform, PlatformEnum
@@ -178,8 +177,6 @@ class XPUPlatform(Platform):
@classmethod
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
parallel_config = vllm_config.parallel_config
# lazy import to avoid circular import
from vllm.config import CUDAGraphMode
@@ -190,6 +187,10 @@ class XPUPlatform(Platform):
attention_config = vllm_config.attention_config
if attention_config.backend is None:
attention_config.backend = AttentionBackendEnum.FLASH_ATTN
# lazy import to avoid circular import
from vllm.utils.torch_utils import supports_xpu_graph
if not supports_xpu_graph():
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
logger.warning(
@@ -324,9 +325,8 @@ class XPUPlatform(Platform):
@classmethod
def get_device_communicator_cls(cls) -> str:
from vllm.utils.torch_utils import supports_xccl
if not supports_xccl():
if not torch.distributed.is_xccl_available():
# Supports xccl with PyTorch versions >= 2.8.0.dev for XPU platform
logger.warning(
"xccl is not enabled in this torch build, communication"
" is not available."
+18 -15
View File
@@ -3,7 +3,6 @@
import contextlib
import importlib.metadata
import os
import platform
import random
import threading
from collections.abc import Callable, Collection
@@ -18,6 +17,7 @@ from torch.library import Library, infer_schema
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.utils.platform_utils import is_pin_memory_available
if TYPE_CHECKING:
from vllm.config import ModelConfig
@@ -68,9 +68,7 @@ MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP = {
T = TypeVar("T")
# Pin memory in non-WSL case.
# Logic duplicated here for now to avoid circular import.
PIN_MEMORY = "microsoft" not in " ".join(platform.uname()).lower()
PIN_MEMORY = is_pin_memory_available()
def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
@@ -606,14 +604,24 @@ def create_kv_caches_with_random(
def async_tensor_h2d(
data: list,
dtype: torch.dtype,
data: list | np.ndarray | torch.Tensor,
device: str | torch.device,
pin_memory: bool = PIN_MEMORY,
dtype: torch.dtype | None = None,
) -> torch.Tensor:
"""Asynchronously create a tensor and copy it from host to device."""
t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
return t.to(device=device, non_blocking=True)
"""Copy list/numpy array/tensor async from host to device."""
if isinstance(data, np.ndarray):
data = torch.from_numpy(data)
if isinstance(data, torch.Tensor):
t = data.pin_memory() if PIN_MEMORY else data
else:
t = torch.tensor(data, dtype=dtype, pin_memory=PIN_MEMORY, device="cpu")
assert t.is_cpu
return t.to(device=device, dtype=dtype, non_blocking=True)
def np_to_pinned_tensor(array: np.ndarray) -> torch.Tensor:
t = torch.from_numpy(array)
return t.pin_memory() if PIN_MEMORY else t
def make_ndarray_with_pad(
@@ -914,11 +922,6 @@ def _encode_layer_name(layer_name: str) -> str | LayerName:
return LayerName(layer_name) if _USE_LAYERNAME else layer_name
# Supports xccl with PyTorch versions >= 2.8.0.dev for XPU platform
def supports_xccl() -> bool:
return torch.distributed.is_xccl_available()
# Supports XPU Graph with PyTorch versions >= 2.11.0.dev for XPU platform
def supports_xpu_graph() -> bool:
return is_torch_equal_or_newer("2.11.0.dev")
+2 -4
View File
@@ -41,8 +41,8 @@ from vllm.utils.flashinfer import (
use_trtllm_attention,
)
from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import (
PIN_MEMORY,
canonicalize_singleton_dim_strides,
is_quantized_kv_cache,
is_strictly_contiguous,
@@ -708,9 +708,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# Since we do not have explicit synchronization in ModelRunnerV2, we do not pin
# reused CPU buffers to avoid a race condition between step N async copies to
# GPU and step N+1 buffer updates.
self.pin_memory = (
not vllm_config.use_v2_model_runner and is_pin_memory_available()
)
self.pin_memory = not vllm_config.use_v2_model_runner and PIN_MEMORY
self.paged_kv_indptr = self._make_buffer(max_num_reqs + 1)
self.paged_kv_indptr_cpu_buffer = torch.zeros_like(
self.paged_kv_indptr.cpu, pin_memory=self.pin_memory
+6 -2
View File
@@ -28,7 +28,11 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import is_quantized_kv_cache, is_torch_equal_or_newer
from vllm.utils.torch_utils import (
async_tensor_h2d,
is_quantized_kv_cache,
is_torch_equal_or_newer,
)
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -58,7 +62,7 @@ def _offsets_to_doc_ids_tensor(
doc_ids = torch.repeat_interleave(
torch.arange(len(counts), dtype=torch.int32), counts
)
return doc_ids.to(device, non_blocking=True)
return async_tensor_h2d(doc_ids, device=device)
def pad_to_multiple(x: torch.Tensor, multiple: int, dim: int):
+11 -8
View File
@@ -8,6 +8,7 @@ from typing import Literal
import torch
from vllm.config import VllmConfig
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -203,8 +204,8 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
spec_sequence_masks = None
spec_sequence_masks_cpu = None
else:
spec_sequence_masks = spec_sequence_masks_cpu.to(
query_start_loc.device, non_blocking=True
spec_sequence_masks = async_tensor_h2d(
spec_sequence_masks_cpu, device=query_start_loc.device
)
if spec_sequence_masks is None:
@@ -376,12 +377,14 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
)
assert prefill_query_start_loc_cpu is not None
chunk_indices = prepare_chunk_indices(
prefill_query_start_loc_cpu, FLA_CHUNK_SIZE
).to(device=gpu_device, non_blocking=True)
chunk_offsets = prepare_chunk_offsets(
prefill_query_start_loc_cpu, FLA_CHUNK_SIZE
).to(device=gpu_device, non_blocking=True)
chunk_indices = async_tensor_h2d(
prepare_chunk_indices(prefill_query_start_loc_cpu, FLA_CHUNK_SIZE),
device=gpu_device,
)
chunk_offsets = async_tensor_h2d(
prepare_chunk_offsets(prefill_query_start_loc_cpu, FLA_CHUNK_SIZE),
device=gpu_device,
)
if num_prefills > 0:
has_initial_state = context_lens_tensor > 0
+14 -13
View File
@@ -7,6 +7,7 @@ from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
@@ -68,22 +69,22 @@ def compute_varlen_chunk_metadata(
# Exclusive prefix sum over logical-chunk lengths
if chunk_lens:
cu_chunk_seqlens = torch.tensor(
[0] + list(itertools.accumulate(chunk_lens)),
device=device,
dtype=torch.int32,
)
# Final boundary must equal total tokens
assert int(cu_chunk_seqlens[-1].item()) == total
cu_chunk_seqlens_list = [0] + list(itertools.accumulate(chunk_lens))
# Final boundary must equal total tokens (check on host to avoid a sync)
assert cu_chunk_seqlens_list[-1] == total
else:
cu_chunk_seqlens = torch.tensor([0], device=device, dtype=torch.int32)
cu_chunk_seqlens_list = [0]
cu_chunk_seqlens = async_tensor_h2d(
cu_chunk_seqlens_list, dtype=torch.int32, device=device
)
last_chunk_indices_t = (
torch.tensor(last_chunk_indices, device=device, dtype=torch.int32)
if len(starts) > 0
else torch.empty((0,), device=device, dtype=torch.int32)
# last_chunk_indices is empty when there are no sequences (len(starts) == 0).
last_chunk_indices_t = async_tensor_h2d(
last_chunk_indices, dtype=torch.int32, device=device
)
seq_idx_chunks_t = async_tensor_h2d(
seq_idx_chunks, dtype=torch.int32, device=device
)
seq_idx_chunks_t = torch.tensor(seq_idx_chunks, device=device, dtype=torch.int32)
return cu_chunk_seqlens, last_chunk_indices_t, seq_idx_chunks_t
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
get_mla_dims,
)
from vllm.platforms.interface import DeviceCapability
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -217,7 +217,7 @@ class FlashInferMLASparseMetadataBuilder(
# Zero-fill for cudagraphs
self.req_id_per_token_buffer.fill_(0)
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
torch.from_numpy(req_id_per_token), non_blocking=True
np_to_pinned_tensor(req_id_per_token), non_blocking=True
)
req_id_per_token_tensor = self.req_id_per_token_buffer[:num_tokens]
@@ -16,7 +16,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -503,7 +503,7 @@ class FlashMLASparseMetadataBuilder(AttentionMetadataBuilder[FlashMLASparseMetad
# Zero-fill for cudagraphs
self.req_id_per_token_buffer.fill_(0)
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
torch.from_numpy(req_id_per_token), non_blocking=True
np_to_pinned_tensor(req_id_per_token), non_blocking=True
)
req_id_per_token = self.req_id_per_token_buffer[:num_tokens]
+16 -19
View File
@@ -17,7 +17,7 @@ from typing_extensions import runtime_checkable
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
if TYPE_CHECKING:
@@ -364,8 +364,8 @@ def make_local_attention_virtual_batches(
# tensor first, which recovers perf.
# Upload the index tensors to the block_table's device up-front so that the
# fancy indexing below doesn't implicitly force a synchronous H2D copy.
batch_indices_torch = torch.from_numpy(batch_indices).to(device, non_blocking=True)
block_indices_torch = torch.from_numpy(block_indices).to(device, non_blocking=True)
batch_indices_torch = async_tensor_h2d(batch_indices, device=device)
block_indices_torch = async_tensor_h2d(block_indices, device=device)
# Save as a lambda so we can return this for update_block_table
make_block_table = lambda block_table: block_table[
@@ -379,8 +379,8 @@ def make_local_attention_virtual_batches(
return CommonAttentionMetadata(
query_start_loc_cpu=query_start_loc_cpu,
query_start_loc=query_start_loc_cpu.to(device=device, non_blocking=True),
seq_lens=seq_lens_cpu.to(device=device, non_blocking=True),
query_start_loc=async_tensor_h2d(query_start_loc_cpu, device=device),
seq_lens=async_tensor_h2d(seq_lens_cpu, device=device),
num_reqs=len(seq_lens_cpu),
num_actual_tokens=common_attn_metadata.num_actual_tokens,
max_query_len=seqlens_q_local.max(),
@@ -808,14 +808,12 @@ def create_fast_prefill_custom_backend(
def compute_causal_conv1d_metadata(
query_start_loc_p_cpu: torch.Tensor,
*,
device: torch.device,
):
query_start_loc_p_cpu: torch.Tensor, *, device: torch.device
) -> tuple[dict[int, dict[str, Any]], torch.Tensor, torch.Tensor]:
# Needed for causal_conv1d. Use the CPU query_start_loc to avoid DtoH sync.
assert query_start_loc_p_cpu.device.type == "cpu"
seqlens = query_start_loc_p_cpu.diff()
nums_dict = {} # type: ignore
nums_dict: dict[int, dict[str, Any]] = {}
batch_ptr = None
token_chunk_offset_ptr = None
for BLOCK_M in [8]: # cover all BLOCK_M values
@@ -823,7 +821,7 @@ def compute_causal_conv1d_metadata(
nums_dict[BLOCK_M] = {}
nums_dict[BLOCK_M]["nums"] = nums
nums_dict[BLOCK_M]["tot"] = nums.sum().item()
mlist = torch.from_numpy(np.repeat(np.arange(len(nums)), nums))
mlist = np_to_pinned_tensor(np.repeat(np.arange(len(nums)), nums))
nums_dict[BLOCK_M]["mlist"] = mlist
mlist_len = len(nums_dict[BLOCK_M]["mlist"])
nums_dict[BLOCK_M]["mlist_len"] = mlist_len
@@ -831,7 +829,7 @@ def compute_causal_conv1d_metadata(
offsetlist = [] # type: ignore
for idx, num in enumerate(nums):
offsetlist.extend(range(num))
offsetlist = torch.tensor(offsetlist, dtype=torch.int32)
offsetlist = torch.tensor(offsetlist, dtype=torch.int32, pin_memory=PIN_MEMORY)
nums_dict[BLOCK_M]["offsetlist"] = offsetlist
if batch_ptr is None:
@@ -845,16 +843,15 @@ def compute_causal_conv1d_metadata(
else:
if batch_ptr.nelement() < MAX_NUM_PROGRAMS:
batch_ptr.resize_(MAX_NUM_PROGRAMS).fill_(PAD_SLOT_ID)
token_chunk_offset_ptr.resize_( # type: ignore
MAX_NUM_PROGRAMS
).fill_(PAD_SLOT_ID)
assert token_chunk_offset_ptr is not None
token_chunk_offset_ptr.resize_(MAX_NUM_PROGRAMS).fill_(PAD_SLOT_ID)
assert batch_ptr is not None
batch_ptr[0:mlist_len].copy_(mlist, non_blocking=True)
token_chunk_offset_ptr[ # type: ignore
0:mlist_len
].copy_(offsetlist, non_blocking=True)
assert token_chunk_offset_ptr is not None
token_chunk_offset_ptr[0:mlist_len].copy_(offsetlist, non_blocking=True)
nums_dict[BLOCK_M]["batch_ptr"] = batch_ptr
nums_dict[BLOCK_M]["token_chunk_offset_ptr"] = token_chunk_offset_ptr # type: ignore
nums_dict[BLOCK_M]["token_chunk_offset_ptr"] = token_chunk_offset_ptr
return nums_dict, batch_ptr, token_chunk_offset_ptr
+3 -3
View File
@@ -14,7 +14,7 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.kv_offload.base import (
BlockIDsLoadStoreSpec,
CanonicalKVCacheRef,
@@ -156,7 +156,7 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None:
def _new_descriptor_buffers(
num_copy_ops: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
pin = is_pin_memory_available()
pin = PIN_MEMORY
# CUDA cache_kernels.cu requires int64; XPU DMA engine requires uint64.
ptr_dtype = torch.uint64 if current_platform.is_xpu() else torch.int64
return (
@@ -482,7 +482,7 @@ class CpuGpuOffloadingHandlers:
num_cpu_blocks: int,
mmap_region: SharedOffloadRegion | None = None,
):
pin_memory = is_pin_memory_available()
pin_memory = PIN_MEMORY
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
self._mmap_region = mmap_region
if mmap_region is not None and pin_memory:
+2 -4
View File
@@ -7,9 +7,7 @@ import torch
from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask
from vllm.utils.platform_utils import is_pin_memory_available
pin_memory = is_pin_memory_available()
from vllm.utils.torch_utils import PIN_MEMORY
@dataclass
@@ -134,7 +132,7 @@ class PoolingMetadata:
num_scheduled_tokens_cpu = torch.from_numpy(num_scheduled_tokens_np)
if query_start_loc_gpu is None:
cumsum = torch.zeros(
n_seq + 1, dtype=torch.int64, pin_memory=pin_memory, device="cpu"
n_seq + 1, dtype=torch.int64, pin_memory=PIN_MEMORY, device="cpu"
)
torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:])
cumsum = cumsum.to(device, non_blocking=True)
+5 -10
View File
@@ -7,6 +7,7 @@ import numpy as np
import torch
from vllm import SamplingParams
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.sample.logits_processor.interface import (
BatchUpdate,
LogitsProcessor,
@@ -118,7 +119,6 @@ class MinPLogitsProcessor(LogitsProcessor):
class LogitBiasLogitsProcessor(LogitsProcessor):
def __init__(self, _, device: torch.device, is_pin_memory: bool):
self.device = device
self.pin_memory = is_pin_memory
self.biases: dict[int, dict[int, float]] = {}
self.bias_tensor: torch.Tensor = torch.tensor(())
@@ -154,9 +154,7 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
)
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
return torch.tensor(
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
).to(device=self.device, non_blocking=True)
return async_tensor_h2d(data, device=self.device, dtype=dtype)
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if self.biases:
@@ -170,7 +168,6 @@ class MinTokensLogitsProcessor(LogitsProcessor):
):
# index -> (min_toks, output_token_ids, stop_token_ids)
self.device = device
self.pin_memory = is_pin_memory
self.min_toks: dict[int, tuple[int, Sequence[int], set[int]]] = {}
# (req_idx_tensor,eos_tok_id_tensor)
@@ -227,9 +224,7 @@ class MinTokensLogitsProcessor(LogitsProcessor):
)
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
return torch.tensor(
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
).to(device=self.device, non_blocking=True)
return async_tensor_h2d(data, device=self.device, dtype=dtype)
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if self.min_toks:
@@ -283,8 +278,8 @@ class MinTokensLogitsProcessor(LogitsProcessor):
toks_arr = np.concatenate(all_toks)
# (row_indices, token_indices) for index_put_ to set -inf.
logits_slice = (
torch.from_numpy(rows_arr).to(self.device, non_blocking=True),
torch.from_numpy(toks_arr).to(self.device, non_blocking=True),
async_tensor_h2d(rows_arr, device=self.device),
async_tensor_h2d(toks_arr, device=self.device),
)
logits.index_put_(logits_slice, self.neg_inf_tensor)
+2 -3
View File
@@ -4,8 +4,7 @@
import torch
from vllm.model_executor.layers.utils import apply_penalties
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import make_tensor_with_pad
from vllm.utils.torch_utils import PIN_MEMORY, make_tensor_with_pad
def apply_all_penalties(
@@ -52,6 +51,6 @@ def _convert_to_tensors(
pad=vocab_size,
device="cpu",
dtype=torch.int64,
pin_memory=is_pin_memory_available(),
pin_memory=PIN_MEMORY,
)
return output_tokens_tensor.to(device, non_blocking=True)
+2 -2
View File
@@ -6,7 +6,7 @@ import torch
import torch.nn as nn
from vllm.config.model import LogprobsMode
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.outputs import LogprobsTensors, SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.ops.bad_words import apply_bad_words
@@ -65,7 +65,7 @@ class Sampler(nn.Module):
):
super().__init__()
self.topk_topp_sampler = TopKTopPSampler(logprobs_mode, use_fp64_gumbel)
self.pin_memory = is_pin_memory_available()
self.pin_memory = PIN_MEMORY
self.logprobs_mode = logprobs_mode
self.use_fp64_gumbel = use_fp64_gumbel
+2 -3
View File
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.sample.logits_processor.interface import (
BatchUpdate,
MoveDirectionality,
@@ -22,12 +22,11 @@ def maybe_create_thinking_budget_state_holder(
max_num_seqs: int,
num_spec_tokens: int,
device: torch.device,
is_pin_memory: bool,
) -> "ThinkingBudgetStateHolder | None":
if reasoning_config is None:
return None
return ThinkingBudgetStateHolder(
reasoning_config, max_num_seqs, num_spec_tokens, device, is_pin_memory
reasoning_config, max_num_seqs, num_spec_tokens, device, PIN_MEMORY
)
+2 -2
View File
@@ -33,7 +33,7 @@ from vllm.multimodal.inputs import (
MultiModalSharedField,
NestedTensors,
)
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.utils import tensor_data
logger = init_logger(__name__)
@@ -327,7 +327,7 @@ class MsgpackDecoder:
oob_tensor_provider: OOBTensorProvider | None = None,
):
self.share_mem = share_mem
self.pin_tensors = is_pin_memory_available()
self.pin_tensors = PIN_MEMORY
args = () if t is None else (t,)
self.decoder = msgpack.Decoder(
*args, ext_hook=self.ext_hook, dec_hook=self.dec_hook
+2 -2
View File
@@ -8,7 +8,7 @@ import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.simple_kv_offload.copy_backend import DmaCopyBackend
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
from vllm.v1.simple_kv_offload.metadata import (
@@ -149,7 +149,7 @@ class SimpleCPUOffloadWorker:
(self.num_cpu_blocks * total_bytes_per_block) / (1024**3),
)
pin_memory = is_pin_memory_available()
pin_memory = PIN_MEMORY
if not pin_memory:
logger.warning(
"Pinned memory not available. CPU offload performance may be degraded."
+2 -3
View File
@@ -12,7 +12,7 @@ from vllm.config import CUDAGraphMode, VllmConfig, get_layers_from_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.model_loader import get_model
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.attention.backend import AttentionMetadataBuilder, CommonAttentionMetadata
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
from vllm.v1.utils import CpuGpuBuffer
@@ -58,7 +58,7 @@ class ExtractHiddenStatesProposer:
self.backup_next_token_ids = CpuGpuBuffer(
max_batch_size,
dtype=torch.int32,
pin_memory=is_pin_memory_available(),
pin_memory=PIN_MEMORY,
device=device,
with_numpy=True,
)
@@ -317,7 +317,6 @@ class ExtractHiddenStatesProposer:
(batch_size, 1). For each request we either use the sampled token
(if valid and not discarded) or a backup token from the request state.
"""
num_reqs = gpu_input_batch.num_reqs
# Precompute backup token IDs for discarded requests.
num_reqs = gpu_input_batch.num_reqs
+7 -9
View File
@@ -26,7 +26,7 @@ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadata
@@ -228,7 +228,7 @@ class SpecDecodeBaseProposer:
self.backup_next_token_ids = CpuGpuBuffer(
self.max_batch_size,
dtype=torch.int32,
pin_memory=is_pin_memory_available(),
pin_memory=PIN_MEMORY,
device=device,
with_numpy=True,
)
@@ -239,9 +239,7 @@ class SpecDecodeBaseProposer:
self._last_draft_probs: torch.Tensor | None = None
self._slot_mapping_buffer = torch.zeros(
self.max_positions,
dtype=torch.int64,
device=device,
self.max_positions, dtype=torch.int64, device=device
)
# Determine allowed attention backends once during initialization.
@@ -1127,7 +1125,7 @@ class SpecDecodeBaseProposer:
new_query_start_loc_cpu = torch.zeros(
query_start_loc_cpu.shape,
dtype=torch.int32,
pin_memory=is_pin_memory_available(),
pin_memory=PIN_MEMORY,
)
new_query_start_loc_np = new_query_start_loc_cpu.numpy()
np.cumsum(new_num_tokens_per_req_np, out=new_query_start_loc_np[1:])
@@ -1160,11 +1158,11 @@ class SpecDecodeBaseProposer:
# q1 + 0, q1 + 1, q1 + 2, q1 + 3, // req 2
# q1 + q2 + 0, q1 + q2 + 1, q1 + q2 + 2] // req 3
token_indices_np = token_offsets + old_query_start_locs_expanded
token_indices = torch.from_numpy(token_indices_np).to(device, non_blocking=True)
token_indices = async_tensor_h2d(token_indices_np, device=device)
spec_common_attn_metadata = CommonAttentionMetadata(
query_start_loc=new_query_start_loc_cpu.to(device, non_blocking=True),
seq_lens=new_seq_lens_cpu.to(device, non_blocking=True),
query_start_loc=async_tensor_h2d(new_query_start_loc_cpu, device=device),
seq_lens=async_tensor_h2d(new_seq_lens_cpu, device=device),
query_start_loc_cpu=new_query_start_loc_cpu,
_seq_lens_cpu=new_seq_lens_cpu,
_num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu,
+2 -2
View File
@@ -545,7 +545,7 @@ def update_ngram_gpu_tensors_incremental(
num_tokens = input_batch.num_tokens_no_spec[idx]
if num_tokens > 0:
token_ids_gpu_tensor[idx, :num_tokens].copy_(
input_batch.token_ids_cpu_tensor[idx, :num_tokens],
input_batch.token_ids_cpu_tensor[idx, :num_tokens].pin_memory(),
non_blocking=True,
)
@@ -591,7 +591,7 @@ def update_ngram_gpu_tensors_incremental(
num_tokens = input_batch.num_tokens_no_spec[new_req_idx]
if num_tokens > 0:
token_ids_gpu_tensor[new_req_idx, :num_tokens].copy_(
input_batch.token_ids_cpu_tensor[new_req_idx, :num_tokens],
input_batch.token_ids_cpu_tensor[new_req_idx, :num_tokens].pin_memory(),
non_blocking=True,
)
@@ -11,7 +11,7 @@ from transformers import PreTrainedTokenizerBase
from vllm.sampling_params import SamplingParams
from vllm.utils.import_utils import LazyLoader
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.structured_output.backend_types import (
StructuredOutputBackend,
StructuredOutputGrammar,
@@ -139,7 +139,7 @@ class LMFormatEnforcerBackend(StructuredOutputBackend):
(max_num_seqs, (self.vocab_size + 31) // 32),
-1,
dtype=torch.int32,
pin_memory=is_pin_memory_available(),
pin_memory=PIN_MEMORY,
)
def destroy(self):
@@ -15,7 +15,7 @@ from regex import escape as regex_escape
from vllm.sampling_params import SamplingParams
from vllm.utils.import_utils import LazyLoader
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.structured_output.backend_types import (
StructuredOutputBackend,
StructuredOutputGrammar,
@@ -101,7 +101,7 @@ class OutlinesBackend(StructuredOutputBackend):
(max_num_seqs, (self.vocab_size + 31) // 32),
-1,
dtype=torch.int32,
pin_memory=is_pin_memory_available(),
pin_memory=PIN_MEMORY,
)
def destroy(self):
+11 -14
View File
@@ -10,7 +10,6 @@ from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from typing import TYPE_CHECKING, TypeVar
import numpy as np
import regex as re
import torch
from cachetools import LRUCache
@@ -18,7 +17,7 @@ from cachetools import LRUCache
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.utils.import_utils import LazyLoader
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
if TYPE_CHECKING:
@@ -123,11 +122,13 @@ def apply_grammar_bitmask(
out_indices = []
# Reorder the bitmask to match the order of the requests in the batch.
sorted_bitmask = np.full(
shape=(logits.shape[0], grammar_bitmask.shape[1]),
fill_value=-1,
dtype=grammar_bitmask.dtype,
sorted_bitmask_tensor = torch.full(
(logits.shape[0], grammar_bitmask.shape[1]),
-1,
dtype=torch.from_numpy(grammar_bitmask[:0]).dtype,
pin_memory=PIN_MEMORY,
)
sorted_bitmask = sorted_bitmask_tensor.numpy()
cumulative_index = 0
for req_id in grammar_output.structured_output_request_ids:
num_spec_tokens = len(spec_tokens.get(req_id, ()))
@@ -138,10 +139,8 @@ def apply_grammar_bitmask(
out_indices.append(bitmask_index)
cumulative_index += 1 + num_spec_tokens
# Copy async to device as tensor.
grammar_bitmask = torch.from_numpy(sorted_bitmask).to(
logits.device, non_blocking=True
)
# Copy async to device.
grammar_bitmask = sorted_bitmask_tensor.to(logits.device, non_blocking=True)
# If the length of out indices and the logits have the same shape
# we don't need to pass indices to the kernel,
@@ -154,11 +153,9 @@ def apply_grammar_bitmask(
# xgrammar expects a python list of indices but it will actually work with
# a tensor. If we copy the tensor ourselves here we can do it in a
# non_blocking manner and there should be no cpu sync within xgrammar.
pin_memory = is_pin_memory_available()
index_tensor = torch.tensor(
out_indices, dtype=torch.int32, device="cpu", pin_memory=pin_memory
index_tensor = async_tensor_h2d(
out_indices, dtype=torch.int32, device=logits.device
)
index_tensor = index_tensor.to(logits.device, non_blocking=True)
xgr.apply_token_bitmask_inplace(logits, grammar_bitmask, indices=index_tensor)
return
+2 -1
View File
@@ -31,6 +31,7 @@ from vllm.logger import init_logger
from vllm.usage.usage_lib import UsageContext, is_usage_stats_enabled, usage_message
from vllm.utils.network_utils import get_open_zmq_ipc_path, get_tcp_uri
from vllm.utils.system_utils import decorate_logs, kill_process_tree, set_process_title
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
@@ -114,7 +115,7 @@ class CpuGpuBuffer:
*size: int | torch.SymInt,
dtype: torch.dtype,
device: torch.device,
pin_memory: bool,
pin_memory: bool = PIN_MEMORY,
with_numpy: bool = True,
) -> None:
# these buffers are mutable runtime state, so allocate them as normal
+8 -3
View File
@@ -7,6 +7,8 @@
from typing import Any
import numpy as np
# Patch torch APIs
import torch
@@ -45,11 +47,14 @@ import vllm.utils.torch_utils as torch_utils
def async_tensor_h2d(
data: list,
dtype: torch.dtype,
data: list | np.ndarray | torch.Tensor,
device: str | torch.device,
pin_memory: bool = False,
dtype: torch.dtype | None = None,
) -> torch.Tensor:
if isinstance(data, np.ndarray):
data = torch.from_numpy(data)
if isinstance(data, torch.Tensor):
return data.to(dtype=dtype)
return torch.tensor(data, dtype=dtype, device="cpu")
+5 -6
View File
@@ -36,10 +36,9 @@ def async_copy_to_gpu(
assert device is not None
out = torch.empty_like(x, device=device)
# Copy directly to GPU — explicit pin_memory() causes sporadic stalls
# under high concurrency due to CUDA driver contention. The driver
# handles the transfer efficiently without manual pinning.
return out.copy_(x, non_blocking=True)
# pin_memory() is no-op if the memory is already pinned.
pinned = x.pin_memory()
return out.copy_(pinned, non_blocking=True)
class UvaBuffer:
@@ -183,7 +182,7 @@ class StagedWriteTensor:
# Special handling for write_contents
write_contents = async_tensor_h2d(
self._staged_write_contents, self.dtype, self.device
self._staged_write_contents, device=self.device, dtype=self.dtype
)
# Write diffs to the GPU buffer
@@ -255,7 +254,7 @@ class FusedStagedWriter:
indices_uva = self.indices.copy_to_uva(indices)
starts_uva = self.starts.copy_to_uva(starts)
cu_lens_uva = self.cu_lens.copy_to_uva(cu_lens)
contents_gpu = async_tensor_h2d(contents, torch.int32, self.device)
contents_gpu = async_tensor_h2d(contents, device=self.device, dtype=torch.int32)
_apply_write_kernel[(len(group_ids),)](
output_ptrs,
+2 -3
View File
@@ -49,12 +49,11 @@ class EncoderRunner:
@torch.inference_mode()
def execute_mm_encoder(
self,
mm_kwargs: list[tuple[str, MultiModalKwargsItem]],
self, mm_kwargs: list[tuple[str, MultiModalKwargsItem]]
) -> list[torch.Tensor]:
encoder_outputs: list[torch.Tensor] = []
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
mm_kwargs, device=self.device, pin_memory=False
mm_kwargs, device=self.device, pin_memory=True
):
batch_outputs = self.model.embed_multimodal(**mm_kwargs_batch)
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
+2 -3
View File
@@ -46,8 +46,7 @@ from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
from vllm.utils.math_utils import cdiv
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.utils.torch_utils import PIN_MEMORY, STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
@@ -498,7 +497,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
self.kv_block_zeroer = KVBlockZeroer(
self.device,
is_pin_memory_available(),
pin_memory=PIN_MEMORY,
attn_groups_iter=(g for groups in self.attn_groups for g in groups),
kernel_block_sizes=self.kernel_block_sizes,
cache_dtype=self.cache_config.cache_dtype,
+3 -4
View File
@@ -170,7 +170,8 @@ class WhisperModelState(ModelState):
for_capture: bool,
num_reqs: int,
) -> dict[int, tuple[torch.Tensor, np.ndarray]]:
encoder_seq_lens_np = np.zeros(num_reqs, dtype=np.int32)
encoder_seq_lens = torch.zeros(num_reqs, dtype=torch.int32, pin_memory=True)
encoder_seq_lens_np = encoder_seq_lens.numpy()
if not for_capture:
# During normal execution, use actual encoder lengths.
for i, req_id in enumerate(req_ids):
@@ -183,9 +184,7 @@ class WhisperModelState(ModelState):
# is captured with the correct value for cross-attention.
encoder_seq_lens_np[:] = self.max_encoder_len
self.encoder_seq_lens_gpu[:num_reqs].copy_(
torch.from_numpy(encoder_seq_lens_np), non_blocking=True
)
self.encoder_seq_lens_gpu[:num_reqs].copy_(encoder_seq_lens, non_blocking=True)
self.encoder_seq_lens_gpu[num_reqs:].fill_(0)
encoder_seq_lens_gpu = self.encoder_seq_lens_gpu[:num_reqs]
+17 -16
View File
@@ -15,6 +15,7 @@ from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams, SamplingType
from vllm.utils import length_from_prompt_token_ids_or_embeds
from vllm.utils.collection_utils import swap_dict_values
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.sample.logits_processor import (
@@ -95,7 +96,6 @@ class InputBatch:
max_model_len: int,
max_num_batched_tokens: int,
device: torch.device,
pin_memory: bool,
vocab_size: int,
block_sizes: list[int], # The block_size of each kv cache group
kernel_block_sizes: list[int],
@@ -112,7 +112,6 @@ class InputBatch:
max_num_reqs,
num_spec_tokens,
device,
pin_memory,
)
self.thinking_token_budget_reqs: set[str] = set()
self.is_pooling_model = is_pooling_model
@@ -120,7 +119,6 @@ class InputBatch:
self.max_model_len = max_model_len
self.max_num_batched_tokens = max_num_batched_tokens
self.device = device
self.pin_memory = pin_memory
self.vocab_size = vocab_size
self._req_ids: list[str | None] = []
@@ -138,7 +136,10 @@ class InputBatch:
)
self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
self.is_token_ids_tensor = torch.zeros(
(max_num_reqs, max_model_len), device="cpu", dtype=bool, pin_memory=False
(max_num_reqs, max_model_len),
device="cpu",
dtype=bool,
pin_memory=False,
)
self.is_token_ids = self.is_token_ids_tensor.numpy()
# Store prompt embeddings per request to avoid OOM from large upfront
@@ -149,21 +150,21 @@ class InputBatch:
(max_num_reqs,),
device="cpu",
dtype=torch.int32,
pin_memory=pin_memory,
pin_memory=PIN_MEMORY,
)
self.num_tokens_no_spec = self.num_tokens_no_spec_cpu_tensor.numpy()
self.num_prompt_tokens_cpu_tensor = torch.zeros(
(max_num_reqs,),
device="cpu",
dtype=torch.int32,
pin_memory=pin_memory,
pin_memory=PIN_MEMORY,
)
self.num_prompt_tokens = self.num_prompt_tokens_cpu_tensor.numpy()
self.num_computed_tokens_cpu_tensor = torch.zeros(
(max_num_reqs,),
device="cpu",
dtype=torch.int32,
pin_memory=pin_memory,
pin_memory=PIN_MEMORY,
)
self.num_computed_tokens_cpu = self.num_computed_tokens_cpu_tensor.numpy()
@@ -172,7 +173,7 @@ class InputBatch:
max_num_reqs=max_num_reqs,
max_model_len=max_model_len,
max_num_batched_tokens=max_num_batched_tokens,
pin_memory=pin_memory,
pin_memory=PIN_MEMORY,
device=device,
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
@@ -185,7 +186,7 @@ class InputBatch:
(max_num_reqs,), dtype=torch.float32, device=device
)
self.temperature_cpu_tensor = torch.empty(
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=PIN_MEMORY
)
self.temperature_cpu = self.temperature_cpu_tensor.numpy()
self.greedy_reqs: set[str] = set()
@@ -193,14 +194,14 @@ class InputBatch:
self.top_p = torch.empty((max_num_reqs,), dtype=torch.float32, device=device)
self.top_p_cpu_tensor = torch.empty(
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=PIN_MEMORY
)
self.top_p_cpu = self.top_p_cpu_tensor.numpy()
self.top_p_reqs: set[str] = set()
self.top_k = torch.empty((max_num_reqs,), dtype=torch.int32, device=device)
self.top_k_cpu_tensor = torch.empty(
(max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=pin_memory
(max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY
)
self.top_k_cpu = self.top_k_cpu_tensor.numpy()
self.top_k_reqs: set[str] = set()
@@ -210,7 +211,7 @@ class InputBatch:
(max_num_reqs,), dtype=torch.float, device=device
)
self.frequency_penalties_cpu_tensor = torch.empty(
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=PIN_MEMORY
)
self.frequency_penalties_cpu = self.frequency_penalties_cpu_tensor.numpy()
self.frequency_penalties_reqs: set[str] = set()
@@ -220,7 +221,7 @@ class InputBatch:
(max_num_reqs,), dtype=torch.float, device=device
)
self.presence_penalties_cpu_tensor = torch.empty(
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=PIN_MEMORY
)
self.presence_penalties_cpu = self.presence_penalties_cpu_tensor.numpy()
self.presence_penalties_reqs: set[str] = set()
@@ -230,14 +231,14 @@ class InputBatch:
(max_num_reqs,), dtype=torch.float, device=device
)
self.repetition_penalties_cpu_tensor = torch.empty(
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=PIN_MEMORY
)
self.repetition_penalties_cpu = self.repetition_penalties_cpu_tensor.numpy()
self.repetition_penalties_reqs: set[str] = set()
# Speculative decoding
self.num_accepted_tokens_cpu_tensor = torch.ones(
(max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=pin_memory
(max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY
)
self.num_accepted_tokens_cpu = self.num_accepted_tokens_cpu_tensor.numpy()
@@ -963,7 +964,7 @@ class InputBatch:
(self.num_reqs, max_prompt_len),
device="cpu",
dtype=torch.int64,
pin_memory=self.pin_memory,
pin_memory=PIN_MEMORY,
)
prompt_token_ids = prompt_token_ids_cpu_tensor.numpy()
prompt_token_ids[:] = self.token_ids_cpu[:num_reqs, :max_prompt_len]
+38 -45
View File
@@ -116,8 +116,10 @@ from vllm.utils import length_from_prompt_token_ids_or_embeds
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.nvtx_pytorch_hooks import PytHooks
from vllm.utils.platform_utils import is_pin_memory_available, num_compute_units
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import (
PIN_MEMORY,
async_tensor_h2d,
get_dtype_size,
is_quantized_kv_cache,
kv_cache_dtype_str_to_dtype,
@@ -441,7 +443,6 @@ class GPUModelRunner(
scheduler_config = self.scheduler_config
parallel_config = self.parallel_config
self.device = device
self.pin_memory = is_pin_memory_available()
self.dtype = self.model_config.dtype
self.kv_cache_dtype = kv_cache_dtype_str_to_dtype(
@@ -666,7 +667,6 @@ class GPUModelRunner(
max_model_len=max(self.max_model_len, self.max_encoder_len),
max_num_batched_tokens=self.max_num_tokens,
device=self.device,
pin_memory=self.pin_memory,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=[placeholder_block_size],
kernel_block_sizes=[placeholder_block_size],
@@ -674,7 +674,7 @@ class GPUModelRunner(
logitsprocs=build_logitsprocs(
self.vllm_config,
self.device,
self.pin_memory,
PIN_MEMORY,
self.is_pooling_model,
custom_logitsprocs,
),
@@ -729,7 +729,7 @@ class GPUModelRunner(
self.max_num_reqs, dtype=torch.int32, device=self.device
)
self.optimistic_seq_lens_cpu = torch.zeros(
self.max_num_reqs, dtype=torch.int32, pin_memory=self.pin_memory
self.max_num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY
)
self.num_computed_tokens = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=self.device
@@ -846,7 +846,7 @@ class GPUModelRunner(
and self.speculative_config.use_ngram_gpu()
):
self._num_valid_draft_tokens_cpu = torch.empty(
self.max_num_reqs, dtype=torch.int32, pin_memory=self.pin_memory
self.max_num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY
)
self._num_valid_draft_tokens_event = torch.cuda.Event()
self._num_valid_draft_tokens_copy_stream = torch.cuda.Stream()
@@ -857,7 +857,7 @@ class GPUModelRunner(
(self.max_num_reqs, 1),
dtype=torch.int64,
device="cpu",
pin_memory=self.pin_memory,
pin_memory=PIN_MEMORY,
)
# Pre-allocated tensor for copying valid sampled token counts to CPU,
@@ -879,7 +879,7 @@ class GPUModelRunner(
(self.max_num_reqs, self.num_spec_tokens),
dtype=torch.int64,
device="cpu",
pin_memory=self.pin_memory,
pin_memory=PIN_MEMORY,
)
if self.use_async_scheduling:
self.valid_sampled_token_count_event = torch.Event()
@@ -888,7 +888,7 @@ class GPUModelRunner(
self.max_num_reqs,
dtype=torch.int32,
device="cpu",
pin_memory=self.pin_memory,
pin_memory=PIN_MEMORY,
)
# Model weight offloader
@@ -1000,7 +1000,6 @@ class GPUModelRunner(
*size,
dtype=dtype,
device=self.device,
pin_memory=self.pin_memory,
with_numpy=numpy,
)
@@ -1055,7 +1054,7 @@ class GPUModelRunner(
token_type_ids.append(ids)
token_type_ids_cpu = torch.empty(
sum(seq_lens_cpu), dtype=torch.int32, pin_memory=self.pin_memory
sum(seq_lens_cpu), dtype=torch.int32, pin_memory=PIN_MEMORY
)
torch.cat(token_type_ids, out=token_type_ids_cpu)
model_kwargs["token_type_ids"] = token_type_ids_cpu.to(
@@ -1095,12 +1094,12 @@ class GPUModelRunner(
"""
self._kv_block_zeroer = KVBlockZeroer(
self.device,
self.pin_memory,
pin_memory=PIN_MEMORY,
attn_groups_iter=self._kv_cache_spec_attn_group_iterator(),
kernel_block_sizes=self._kernel_block_sizes,
cache_dtype=self.cache_config.cache_dtype,
runner_only_attn_layers=self.runner_only_attn_layers,
static_forward_context=(self.compilation_config.static_forward_context),
static_forward_context=self.compilation_config.static_forward_context,
)
def _zero_block_ids(self, block_ids: list[int]) -> None:
@@ -1651,7 +1650,7 @@ class GPUModelRunner(
for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
mm_kwargs,
device=self.device,
pin_memory=self.pin_memory,
pin_memory=PIN_MEMORY,
):
mm_kwargs_combined.update(mm_kwargs_batch)
@@ -1807,10 +1806,10 @@ class GPUModelRunner(
return
# Upload the index tensors asynchronously so the scatter can be non-blocking.
sampled_tokens_index_tensor = torch.tensor(
sample_flattened_indices, dtype=torch.int64, pin_memory=self.pin_memory
sample_flattened_indices, dtype=torch.int64, pin_memory=PIN_MEMORY
).to(self.device, non_blocking=True)
prev_common_req_indices_tensor = torch.tensor(
prev_indices, dtype=torch.int64, pin_memory=self.pin_memory
prev_indices, dtype=torch.int64, pin_memory=PIN_MEMORY
).to(self.device, non_blocking=True)
self.input_ids.gpu.scatter_(
dim=0,
@@ -1826,10 +1825,10 @@ class GPUModelRunner(
assert isinstance(self._draft_token_ids, torch.Tensor)
draft_tokens_index_tensor = torch.tensor(
spec_flattened_indices, dtype=torch.int64, pin_memory=self.pin_memory
spec_flattened_indices, dtype=torch.int64, pin_memory=PIN_MEMORY
).to(self.device, non_blocking=True)
prev_draft_token_indices_tensor = torch.tensor(
prev_draft_token_indices, dtype=torch.int64, pin_memory=self.pin_memory
prev_draft_token_indices, dtype=torch.int64, pin_memory=PIN_MEMORY
).to(self.device, non_blocking=True)
# because input_ids dtype is torch.int32,
@@ -2788,21 +2787,16 @@ class GPUModelRunner(
# [0, 1, 2, 5, 6, 9]
target_logits_indices += self._arange_scratch[: cu_num_draft_tokens[-1]]
# TODO: Optimize the CPU -> GPU copy.
cu_num_draft_tokens = torch.from_numpy(cu_num_draft_tokens).to(
self.device, non_blocking=True
cu_num_draft_tokens = async_tensor_h2d(cu_num_draft_tokens, device=self.device)
cu_num_sampled_tokens = async_tensor_h2d(
cu_num_sampled_tokens, device=self.device
)
cu_num_sampled_tokens = torch.from_numpy(cu_num_sampled_tokens).to(
self.device, non_blocking=True
logits_indices = async_tensor_h2d(logits_indices, device=self.device)
target_logits_indices = async_tensor_h2d(
target_logits_indices, device=self.device
)
logits_indices = torch.from_numpy(logits_indices).to(
self.device, non_blocking=True
)
target_logits_indices = torch.from_numpy(target_logits_indices).to(
self.device, non_blocking=True
)
bonus_logits_indices = torch.from_numpy(bonus_logits_indices).to(
self.device, non_blocking=True
bonus_logits_indices = async_tensor_h2d(
bonus_logits_indices, device=self.device
)
# Compute the draft token ids.
@@ -3012,9 +3006,7 @@ class GPUModelRunner(
# Track the current index in mm_kwargs/mm_lora_refs to map groups to request IDs
current_item_idx = 0
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
mm_kwargs,
device=self.device,
pin_memory=self.pin_memory,
mm_kwargs, device=self.device, pin_memory=PIN_MEMORY
):
batch_outputs: MultiModalEmbeddings
@@ -3048,7 +3040,7 @@ class GPUModelRunner(
group_and_batch_mm_kwargs(
[video_mm_kwargs_item],
device=self.device,
pin_memory=self.pin_memory,
pin_memory=PIN_MEMORY,
)
)
@@ -3107,7 +3099,10 @@ class GPUModelRunner(
mm_embeds = list[torch.Tensor]()
is_mm_embed = torch.zeros(
total_num_scheduled_tokens, dtype=torch.bool, device="cpu"
total_num_scheduled_tokens,
dtype=torch.bool,
device="cpu",
pin_memory=PIN_MEMORY,
)
req_start_idx = 0
@@ -3515,8 +3510,7 @@ class GPUModelRunner(
token_ids_idx_np = np.nonzero(is_token_ids)[0]
# Some tokens ids may need to become embeds
if token_ids_idx_np.size > 0:
token_ids_idx = torch.from_numpy(token_ids_idx_np)
token_ids_idx = token_ids_idx.to(self.device, non_blocking=True)
token_ids_idx = async_tensor_h2d(token_ids_idx_np, device=self.device)
token_ids = self.input_ids.gpu[token_ids_idx]
tokens_to_embeds = self.model.embed_input_ids(input_ids=token_ids)
self.inputs_embeds.gpu[token_ids_idx] = tokens_to_embeds
@@ -4953,7 +4947,7 @@ class GPUModelRunner(
):
indices.append(offset + len(tokens) - 1)
offset += num_draft + 1
indices = torch.tensor(indices, device=self.device)
indices = async_tensor_h2d(indices, device=self.device)
hidden_states = sample_hidden_states[indices]
draft_token_ids = self.drafter.propose(
@@ -5483,8 +5477,8 @@ class GPUModelRunner(
continue
num_prompt_tokens = len(request.prompt_token_ids)
prompt_token_ids = torch.tensor(request.prompt_token_ids).to(
self.device, non_blocking=True
prompt_token_ids = async_tensor_h2d(
request.prompt_token_ids, device=self.device
)
# Set up target LogprobsTensors object.
@@ -5651,7 +5645,7 @@ class GPUModelRunner(
for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
[(modality, dummy_mm_item)] * max_items_per_batch,
device=self.device,
pin_memory=self.pin_memory,
pin_memory=PIN_MEMORY,
)
)
@@ -6995,7 +6989,6 @@ class GPUModelRunner(
max_model_len=max_model_len,
max_num_batched_tokens=self.max_num_tokens,
device=self.device,
pin_memory=self.pin_memory,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
@@ -7430,7 +7423,7 @@ class GPUModelRunner(
self.routed_experts_capturer.device_buffer.shape,
dtype=self.routed_experts_capturer.device_buffer.dtype,
device="cpu",
pin_memory=self.pin_memory,
pin_memory=PIN_MEMORY,
)
# ``slot_mapping`` dtype is fixed to int64 by
# ``block_table.slot_mapping``; we mirror that here.
@@ -7439,7 +7432,7 @@ class GPUModelRunner(
(max_tokens,),
dtype=torch.int64,
device="cpu",
pin_memory=self.pin_memory,
pin_memory=PIN_MEMORY,
)
# Private device buffer so the shared ``block_table.slot_mapping``
# can be overwritten by the next ``_prepare_inputs`` while the