[Core] Ensure memory is pinned prior to async h2d copy (#45424)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -145,7 +145,6 @@ def _generate_fake_sampling_metadata(
|
||||
vllm_config.scheduler_config.max_num_seqs,
|
||||
num_spec,
|
||||
device,
|
||||
PIN_MEMORY_AVAILABLE,
|
||||
)
|
||||
fake_sampling_metadata = SamplingMetadata(
|
||||
temperature=torch.full((batch_size,), 0.0),
|
||||
@@ -880,7 +879,6 @@ def test_maybe_create_thinking_budget_holder_without_reasoning():
|
||||
cfg.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
@@ -35,7 +35,6 @@ def mock_model_runner_with_input_batch():
|
||||
max_model_len=1024,
|
||||
max_num_batched_tokens=1024,
|
||||
device="cpu",
|
||||
pin_memory=False,
|
||||
vocab_size=32000,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
|
||||
@@ -10,7 +10,6 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import make_tensor_with_pad
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessors
|
||||
@@ -236,7 +235,6 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
|
||||
max_model_len=1024,
|
||||
max_num_batched_tokens=1024,
|
||||
device=torch.device(device),
|
||||
pin_memory=is_pin_memory_available(),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
@@ -331,7 +329,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
max_model_len=1024,
|
||||
max_num_batched_tokens=1024,
|
||||
device=torch.device(device),
|
||||
pin_memory=is_pin_memory_available(),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
@@ -341,7 +338,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
max_model_len=1024,
|
||||
max_num_batched_tokens=1024,
|
||||
device=torch.device(device),
|
||||
pin_memory=is_pin_memory_available(),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
@@ -410,7 +406,6 @@ def test_pooling_prompt_lens_not_aliased(device: str):
|
||||
max_model_len=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS,
|
||||
max_num_batched_tokens=batch_size * (MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS),
|
||||
device=torch.device(device),
|
||||
pin_memory=is_pin_memory_available(),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
@@ -459,7 +454,6 @@ def test_pooling_metadata_token_id_buffers(
|
||||
max_model_len=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS,
|
||||
max_num_batched_tokens=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS,
|
||||
device=torch.device("cpu"),
|
||||
pin_memory=False,
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
|
||||
@@ -85,7 +85,6 @@ def initialize_kv_cache(runner: GPUModelRunner):
|
||||
max_model_len=runner.max_model_len,
|
||||
max_num_batched_tokens=runner.max_num_tokens,
|
||||
device=runner.device,
|
||||
pin_memory=runner.pin_memory,
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[
|
||||
@@ -1405,7 +1404,6 @@ def test_input_batch_with_kernel_block_sizes():
|
||||
max_model_len = 512
|
||||
max_num_batched_tokens = 512
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
pin_memory = False
|
||||
vocab_size = 50272
|
||||
|
||||
# Test with different kernel block sizes
|
||||
@@ -1417,7 +1415,6 @@ def test_input_batch_with_kernel_block_sizes():
|
||||
max_model_len=max_model_len,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
vocab_size=vocab_size,
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
@@ -1478,7 +1475,6 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
|
||||
max_model_len=runner.max_model_len,
|
||||
max_num_batched_tokens=runner.max_num_tokens,
|
||||
device=runner.device,
|
||||
pin_memory=runner.pin_memory,
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[16],
|
||||
|
||||
@@ -18,8 +18,8 @@ import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.system_utils import find_loaded_library
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -196,7 +196,7 @@ class CuMemAllocator:
|
||||
size_in_bytes,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
cpu_ptr = cpu_backup_tensor.data_ptr()
|
||||
libcudart.cudaMemcpy(cpu_ptr, ptr, size_in_bytes)
|
||||
|
||||
@@ -11,7 +11,7 @@ import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -188,7 +188,7 @@ class XpuMemAllocator:
|
||||
size_in_bytes,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
cpu_ptr = cpu_backup_tensor.data_ptr()
|
||||
_xpu_memcpy_sync(
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.lora.utils import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -126,7 +126,7 @@ class LoRAModel:
|
||||
skip_prefixes: list[str] | None = None,
|
||||
) -> "LoRAModel":
|
||||
"""Create a LoRAModel from a dictionary of tensors."""
|
||||
pin_memory = str(device) == "cpu" and is_pin_memory_available()
|
||||
pin_memory = str(device) == "cpu" and PIN_MEMORY
|
||||
loras: dict[str, LoRALayerWeights] = {}
|
||||
for tensor_name, tensor in tensors.items():
|
||||
if is_base_embedding_weights(tensor_name):
|
||||
|
||||
@@ -7,7 +7,7 @@ import torch
|
||||
import torch.types
|
||||
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
|
||||
class LoRALayerWeights:
|
||||
@@ -79,7 +79,7 @@ class LoRALayerWeights:
|
||||
dtype: torch.dtype,
|
||||
device: torch.types.Device,
|
||||
) -> "LoRALayerWeights":
|
||||
pin_memory = str(device) == "cpu" and is_pin_memory_available()
|
||||
pin_memory = str(device) == "cpu" and PIN_MEMORY
|
||||
lora_a = torch.zeros(
|
||||
[rank, input_dim], dtype=dtype, device=device, pin_memory=pin_memory
|
||||
)
|
||||
|
||||
@@ -42,7 +42,7 @@ from vllm.model_executor.models.utils import PPMissingLayer
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.encoder_budget import MultiModalBudget
|
||||
from vllm.utils.cache import LRUCache
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -801,7 +801,7 @@ class LoRAModelManager:
|
||||
# 2. The weight packing above (e.g., pack_moe) may invalidate the
|
||||
# pin_memory allocation, so we execute it after packing.
|
||||
|
||||
pin_memory = str(lora_device) == "cpu" and is_pin_memory_available()
|
||||
pin_memory = str(lora_device) == "cpu" and PIN_MEMORY
|
||||
if pin_memory:
|
||||
for lora in lora_model.loras.values():
|
||||
if isinstance(lora.lora_a, list):
|
||||
|
||||
@@ -1684,12 +1684,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
# [[0, 0, 0, 0], [256, 256, 256, 256], [512, 512, 512, 512]]
|
||||
# Note(simon): this is done in CPU because of downstream's
|
||||
# of `to_list`.
|
||||
chunk_starts = (
|
||||
chunk_starts = torch.empty(
|
||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||
).copy_(
|
||||
torch.arange(num_chunks, dtype=torch.int32)
|
||||
.multiply_(max_context_chunk)
|
||||
.unsqueeze(1)
|
||||
.expand(-1, num_prefills)
|
||||
* max_context_chunk
|
||||
).pin_memory()
|
||||
)
|
||||
chunk_ends = torch.min(
|
||||
context_lens_cpu.unsqueeze(0), chunk_starts + max_context_chunk
|
||||
)
|
||||
@@ -1746,12 +1747,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
)
|
||||
* self.dcp_local_block_size
|
||||
)
|
||||
local_chunk_starts = (
|
||||
local_chunk_starts = torch.empty(
|
||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||
).copy_(
|
||||
torch.arange(num_chunks, dtype=torch.int32)
|
||||
.multiply_(padded_local_max_context_chunk_across_ranks)
|
||||
.unsqueeze(1)
|
||||
.expand(-1, num_prefills)
|
||||
* padded_local_max_context_chunk_across_ranks
|
||||
).pin_memory()
|
||||
)
|
||||
local_chunk_ends = torch.min(
|
||||
padded_local_context_lens_cpu.unsqueeze(0),
|
||||
local_chunk_starts
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.utils.flashinfer import (
|
||||
is_flashinfer_cudnn_fp8_prefill_attn_supported,
|
||||
)
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.ops.vit_attn_wrappers import (
|
||||
@@ -311,7 +312,7 @@ class MMEncoderAttention(CustomOp):
|
||||
)
|
||||
cu_seqlens = np.concatenate([cu_seqlens_qko, cu_seqlens_v])
|
||||
|
||||
cu_seqlens = torch.from_numpy(cu_seqlens).to(device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, device=device)
|
||||
return cu_seqlens
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -10,6 +10,7 @@ import torch.nn as nn
|
||||
from vllm.config.pooler import SequencePoolingType
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
SequencePoolingMethodOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
|
||||
@@ -74,15 +75,14 @@ class MeanPool(SequencePoolingMethod):
|
||||
# early return for empty batch
|
||||
return hidden_states.new_empty((0, hidden_size), dtype=torch.float32)
|
||||
|
||||
# Build segment_ids on CPU so repeat_interleave doesn't need to sync
|
||||
# GPU->CPU to learn its data-dependent output length, then upload
|
||||
# non-blocking. eg. [2, 1, 3] -> [0, 0, 1, 2, 2, 2]
|
||||
prompt_lens = async_tensor_h2d(
|
||||
prompt_lens_cpu, device=hidden_states.device, dtype=torch.int64
|
||||
)
|
||||
# eg. [2, 1, 3] -> [0, 0, 1, 2, 2, 2]
|
||||
segment_ids = torch.repeat_interleave(
|
||||
torch.arange(num_seqs, dtype=torch.long),
|
||||
prompt_lens_cpu,
|
||||
).to(hidden_states.device, non_blocking=True)
|
||||
prompt_lens = prompt_lens_cpu.to(
|
||||
hidden_states.device, dtype=torch.int64, non_blocking=True
|
||||
torch.arange(num_seqs, device=hidden_states.device, dtype=torch.long),
|
||||
prompt_lens,
|
||||
output_size=int(prompt_lens_cpu.sum()),
|
||||
)
|
||||
segment_sums = torch.zeros(
|
||||
(num_seqs, hidden_size),
|
||||
|
||||
@@ -66,6 +66,7 @@ from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.vision import is_vit_use_data_parallel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.moonvit import MoonViTConfig
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
def _apply_rope_input_validation(x, freqs_cis):
|
||||
@@ -758,7 +759,7 @@ class MoonVitPretrainedModel(PreTrainedModel):
|
||||
),
|
||||
]
|
||||
)
|
||||
metadata["cu_seqlens"] = torch.from_numpy(cu_seqlens_np).to(device)
|
||||
metadata["cu_seqlens"] = async_tensor_h2d(cu_seqlens_np, device=device)
|
||||
|
||||
if max_seqlen_override is not None:
|
||||
max_seqlen_val = int(max_seqlen_override)
|
||||
@@ -770,7 +771,7 @@ class MoonVitPretrainedModel(PreTrainedModel):
|
||||
metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)
|
||||
|
||||
gather_idx_np = _build_merge_gather_idx(grid_pairs, self.merge_kernel_size)
|
||||
metadata["merge_gather_idx"] = torch.from_numpy(gather_idx_np).to(device)
|
||||
metadata["merge_gather_idx"] = async_tensor_h2d(gather_idx_np, device=device)
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
@@ -83,9 +83,8 @@ from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||
|
||||
@@ -825,7 +824,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
@staticmethod
|
||||
def invert_permutation(perm: torch.Tensor) -> torch.Tensor:
|
||||
# building the inverse permutation in O(n) time
|
||||
inv = torch.empty_like(perm, pin_memory=is_pin_memory_available())
|
||||
inv = torch.empty_like(perm, pin_memory=PIN_MEMORY)
|
||||
inv[perm] = torch.arange(perm.numel(), device=perm.device, dtype=perm.dtype)
|
||||
return inv
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.config.cache import CacheDType
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -207,7 +208,7 @@ class DeepseekV4FlashMLAMetadataBuilder(
|
||||
# Zero-fill for cudagraphs
|
||||
self.req_id_per_token_buffer.fill_(0)
|
||||
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
|
||||
torch.from_numpy(req_id_per_token), non_blocking=True
|
||||
np_to_pinned_tensor(req_id_per_token), non_blocking=True
|
||||
)
|
||||
req_id_per_token = self.req_id_per_token_buffer[:num_tokens]
|
||||
|
||||
|
||||
@@ -488,7 +488,13 @@ class MultiModalBatchedField(BaseMultiModalField):
|
||||
# An optimization when `batch` contains only one tensor:
|
||||
# - produce exactly same result as `torch.stack(batch)`
|
||||
# - will achieve zero-copy if the tensor is contiguous
|
||||
return batch[0].unsqueeze(0).contiguous()
|
||||
out = batch[0].unsqueeze(0)
|
||||
if not pin_memory:
|
||||
return out.contiguous()
|
||||
# Avoid extra copy - pinning unpinned memory will make it contiguous
|
||||
if not out.is_contiguous() and out.is_pinned():
|
||||
out = out.contiguous()
|
||||
return out.pin_memory()
|
||||
first_shape = batch[0].shape
|
||||
if all(elem.shape == first_shape for elem in batch):
|
||||
out = torch.empty(
|
||||
@@ -538,7 +544,13 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
# An optimization when `batch` contains only one tensor:
|
||||
# - produce exactly same result as `torch.concat(batch)`
|
||||
# - will achieve zero-copy if the tensor is contiguous
|
||||
return batch[0].contiguous()
|
||||
out = batch[0]
|
||||
if not pin_memory:
|
||||
return out.contiguous()
|
||||
# Avoid extra copy - pinning unpinned memory will make it contiguous
|
||||
if not out.is_contiguous() and out.is_pinned():
|
||||
out = out.contiguous()
|
||||
return out.pin_memory()
|
||||
|
||||
dim = self.dim + (self.dim < 0) * len(batch[0].shape)
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from typing import TYPE_CHECKING
|
||||
from vllm import envs
|
||||
from vllm.plugins import PLATFORM_PLUGINS_GROUP, load_plugins_by_group
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
from vllm.utils.torch_utils import supports_xccl
|
||||
|
||||
from .interface import CpuArchEnum, Platform, PlatformEnum
|
||||
|
||||
@@ -135,7 +134,7 @@ def xpu_platform_plugin() -> str | None:
|
||||
try:
|
||||
import torch
|
||||
|
||||
if supports_xccl():
|
||||
if torch.distributed.is_xccl_available():
|
||||
dist_backend = "xccl"
|
||||
from vllm.platforms.xpu import XPUPlatform
|
||||
|
||||
|
||||
@@ -23,7 +23,6 @@ import vllm._C_stable_libtorch # noqa
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import import_pynvml
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interface import DeviceCapability, Platform, PlatformEnum, in_wsl
|
||||
@@ -88,6 +87,8 @@ def _get_backend_priorities(
|
||||
kv_cache_dtype: CacheDType | None = None,
|
||||
) -> list[AttentionBackendEnum]:
|
||||
"""Get backend priorities with lazy import to avoid circular dependency."""
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
|
||||
if use_mla:
|
||||
if device_capability.major == 10:
|
||||
# Sparse MLA backend priorities
|
||||
|
||||
@@ -14,7 +14,6 @@ import vllm_xpu_kernels._xpu_C # noqa
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import supports_xpu_graph
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interface import DeviceCapability, Platform, PlatformEnum
|
||||
@@ -178,8 +177,6 @@ class XPUPlatform(Platform):
|
||||
|
||||
@classmethod
|
||||
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
|
||||
# lazy import to avoid circular import
|
||||
from vllm.config import CUDAGraphMode
|
||||
|
||||
@@ -190,6 +187,10 @@ class XPUPlatform(Platform):
|
||||
attention_config = vllm_config.attention_config
|
||||
if attention_config.backend is None:
|
||||
attention_config.backend = AttentionBackendEnum.FLASH_ATTN
|
||||
|
||||
# lazy import to avoid circular import
|
||||
from vllm.utils.torch_utils import supports_xpu_graph
|
||||
|
||||
if not supports_xpu_graph():
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
logger.warning(
|
||||
@@ -324,9 +325,8 @@ class XPUPlatform(Platform):
|
||||
|
||||
@classmethod
|
||||
def get_device_communicator_cls(cls) -> str:
|
||||
from vllm.utils.torch_utils import supports_xccl
|
||||
|
||||
if not supports_xccl():
|
||||
if not torch.distributed.is_xccl_available():
|
||||
# Supports xccl with PyTorch versions >= 2.8.0.dev for XPU platform
|
||||
logger.warning(
|
||||
"xccl is not enabled in this torch build, communication"
|
||||
" is not available."
|
||||
|
||||
+18
-15
@@ -3,7 +3,6 @@
|
||||
import contextlib
|
||||
import importlib.metadata
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import threading
|
||||
from collections.abc import Callable, Collection
|
||||
@@ -18,6 +17,7 @@ from torch.library import Library, infer_schema
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
@@ -68,9 +68,7 @@ MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP = {
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
# Pin memory in non-WSL case.
|
||||
# Logic duplicated here for now to avoid circular import.
|
||||
PIN_MEMORY = "microsoft" not in " ".join(platform.uname()).lower()
|
||||
PIN_MEMORY = is_pin_memory_available()
|
||||
|
||||
|
||||
def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
@@ -606,14 +604,24 @@ def create_kv_caches_with_random(
|
||||
|
||||
|
||||
def async_tensor_h2d(
|
||||
data: list,
|
||||
dtype: torch.dtype,
|
||||
data: list | np.ndarray | torch.Tensor,
|
||||
device: str | torch.device,
|
||||
pin_memory: bool = PIN_MEMORY,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Asynchronously create a tensor and copy it from host to device."""
|
||||
t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
|
||||
return t.to(device=device, non_blocking=True)
|
||||
"""Copy list/numpy array/tensor async from host to device."""
|
||||
if isinstance(data, np.ndarray):
|
||||
data = torch.from_numpy(data)
|
||||
if isinstance(data, torch.Tensor):
|
||||
t = data.pin_memory() if PIN_MEMORY else data
|
||||
else:
|
||||
t = torch.tensor(data, dtype=dtype, pin_memory=PIN_MEMORY, device="cpu")
|
||||
assert t.is_cpu
|
||||
return t.to(device=device, dtype=dtype, non_blocking=True)
|
||||
|
||||
|
||||
def np_to_pinned_tensor(array: np.ndarray) -> torch.Tensor:
|
||||
t = torch.from_numpy(array)
|
||||
return t.pin_memory() if PIN_MEMORY else t
|
||||
|
||||
|
||||
def make_ndarray_with_pad(
|
||||
@@ -914,11 +922,6 @@ def _encode_layer_name(layer_name: str) -> str | LayerName:
|
||||
return LayerName(layer_name) if _USE_LAYERNAME else layer_name
|
||||
|
||||
|
||||
# Supports xccl with PyTorch versions >= 2.8.0.dev for XPU platform
|
||||
def supports_xccl() -> bool:
|
||||
return torch.distributed.is_xccl_available()
|
||||
|
||||
|
||||
# Supports XPU Graph with PyTorch versions >= 2.11.0.dev for XPU platform
|
||||
def supports_xpu_graph() -> bool:
|
||||
return is_torch_equal_or_newer("2.11.0.dev")
|
||||
|
||||
@@ -41,8 +41,8 @@ from vllm.utils.flashinfer import (
|
||||
use_trtllm_attention,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
canonicalize_singleton_dim_strides,
|
||||
is_quantized_kv_cache,
|
||||
is_strictly_contiguous,
|
||||
@@ -708,9 +708,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
# Since we do not have explicit synchronization in ModelRunnerV2, we do not pin
|
||||
# reused CPU buffers to avoid a race condition between step N async copies to
|
||||
# GPU and step N+1 buffer updates.
|
||||
self.pin_memory = (
|
||||
not vllm_config.use_v2_model_runner and is_pin_memory_available()
|
||||
)
|
||||
self.pin_memory = not vllm_config.use_v2_model_runner and PIN_MEMORY
|
||||
self.paged_kv_indptr = self._make_buffer(max_num_reqs + 1)
|
||||
self.paged_kv_indptr_cpu_buffer = torch.zeros_like(
|
||||
self.paged_kv_indptr.cpu, pin_memory=self.pin_memory
|
||||
|
||||
@@ -28,7 +28,11 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache, is_torch_equal_or_newer
|
||||
from vllm.utils.torch_utils import (
|
||||
async_tensor_h2d,
|
||||
is_quantized_kv_cache,
|
||||
is_torch_equal_or_newer,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -58,7 +62,7 @@ def _offsets_to_doc_ids_tensor(
|
||||
doc_ids = torch.repeat_interleave(
|
||||
torch.arange(len(counts), dtype=torch.int32), counts
|
||||
)
|
||||
return doc_ids.to(device, non_blocking=True)
|
||||
return async_tensor_h2d(doc_ids, device=device)
|
||||
|
||||
|
||||
def pad_to_multiple(x: torch.Tensor, multiple: int, dim: int):
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Literal
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -203,8 +204,8 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
spec_sequence_masks = None
|
||||
spec_sequence_masks_cpu = None
|
||||
else:
|
||||
spec_sequence_masks = spec_sequence_masks_cpu.to(
|
||||
query_start_loc.device, non_blocking=True
|
||||
spec_sequence_masks = async_tensor_h2d(
|
||||
spec_sequence_masks_cpu, device=query_start_loc.device
|
||||
)
|
||||
|
||||
if spec_sequence_masks is None:
|
||||
@@ -376,12 +377,14 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
)
|
||||
|
||||
assert prefill_query_start_loc_cpu is not None
|
||||
chunk_indices = prepare_chunk_indices(
|
||||
prefill_query_start_loc_cpu, FLA_CHUNK_SIZE
|
||||
).to(device=gpu_device, non_blocking=True)
|
||||
chunk_offsets = prepare_chunk_offsets(
|
||||
prefill_query_start_loc_cpu, FLA_CHUNK_SIZE
|
||||
).to(device=gpu_device, non_blocking=True)
|
||||
chunk_indices = async_tensor_h2d(
|
||||
prepare_chunk_indices(prefill_query_start_loc_cpu, FLA_CHUNK_SIZE),
|
||||
device=gpu_device,
|
||||
)
|
||||
chunk_offsets = async_tensor_h2d(
|
||||
prepare_chunk_offsets(prefill_query_start_loc_cpu, FLA_CHUNK_SIZE),
|
||||
device=gpu_device,
|
||||
)
|
||||
|
||||
if num_prefills > 0:
|
||||
has_initial_state = context_lens_tensor > 0
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
CommonAttentionMetadata,
|
||||
@@ -68,22 +69,22 @@ def compute_varlen_chunk_metadata(
|
||||
|
||||
# Exclusive prefix sum over logical-chunk lengths
|
||||
if chunk_lens:
|
||||
cu_chunk_seqlens = torch.tensor(
|
||||
[0] + list(itertools.accumulate(chunk_lens)),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
# Final boundary must equal total tokens
|
||||
assert int(cu_chunk_seqlens[-1].item()) == total
|
||||
cu_chunk_seqlens_list = [0] + list(itertools.accumulate(chunk_lens))
|
||||
# Final boundary must equal total tokens (check on host to avoid a sync)
|
||||
assert cu_chunk_seqlens_list[-1] == total
|
||||
else:
|
||||
cu_chunk_seqlens = torch.tensor([0], device=device, dtype=torch.int32)
|
||||
cu_chunk_seqlens_list = [0]
|
||||
cu_chunk_seqlens = async_tensor_h2d(
|
||||
cu_chunk_seqlens_list, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
last_chunk_indices_t = (
|
||||
torch.tensor(last_chunk_indices, device=device, dtype=torch.int32)
|
||||
if len(starts) > 0
|
||||
else torch.empty((0,), device=device, dtype=torch.int32)
|
||||
# last_chunk_indices is empty when there are no sequences (len(starts) == 0).
|
||||
last_chunk_indices_t = async_tensor_h2d(
|
||||
last_chunk_indices, dtype=torch.int32, device=device
|
||||
)
|
||||
seq_idx_chunks_t = async_tensor_h2d(
|
||||
seq_idx_chunks, dtype=torch.int32, device=device
|
||||
)
|
||||
seq_idx_chunks_t = torch.tensor(seq_idx_chunks, device=device, dtype=torch.int32)
|
||||
return cu_chunk_seqlens, last_chunk_indices_t, seq_idx_chunks_t
|
||||
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
get_mla_dims,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -217,7 +217,7 @@ class FlashInferMLASparseMetadataBuilder(
|
||||
# Zero-fill for cudagraphs
|
||||
self.req_id_per_token_buffer.fill_(0)
|
||||
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
|
||||
torch.from_numpy(req_id_per_token), non_blocking=True
|
||||
np_to_pinned_tensor(req_id_per_token), non_blocking=True
|
||||
)
|
||||
req_id_per_token_tensor = self.req_id_per_token_buffer[:num_tokens]
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -503,7 +503,7 @@ class FlashMLASparseMetadataBuilder(AttentionMetadataBuilder[FlashMLASparseMetad
|
||||
# Zero-fill for cudagraphs
|
||||
self.req_id_per_token_buffer.fill_(0)
|
||||
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
|
||||
torch.from_numpy(req_id_per_token), non_blocking=True
|
||||
np_to_pinned_tensor(req_id_per_token), non_blocking=True
|
||||
)
|
||||
req_id_per_token = self.req_id_per_token_buffer[:num_tokens]
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from typing_extensions import runtime_checkable
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -364,8 +364,8 @@ def make_local_attention_virtual_batches(
|
||||
# tensor first, which recovers perf.
|
||||
# Upload the index tensors to the block_table's device up-front so that the
|
||||
# fancy indexing below doesn't implicitly force a synchronous H2D copy.
|
||||
batch_indices_torch = torch.from_numpy(batch_indices).to(device, non_blocking=True)
|
||||
block_indices_torch = torch.from_numpy(block_indices).to(device, non_blocking=True)
|
||||
batch_indices_torch = async_tensor_h2d(batch_indices, device=device)
|
||||
block_indices_torch = async_tensor_h2d(block_indices, device=device)
|
||||
|
||||
# Save as a lambda so we can return this for update_block_table
|
||||
make_block_table = lambda block_table: block_table[
|
||||
@@ -379,8 +379,8 @@ def make_local_attention_virtual_batches(
|
||||
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
query_start_loc=query_start_loc_cpu.to(device=device, non_blocking=True),
|
||||
seq_lens=seq_lens_cpu.to(device=device, non_blocking=True),
|
||||
query_start_loc=async_tensor_h2d(query_start_loc_cpu, device=device),
|
||||
seq_lens=async_tensor_h2d(seq_lens_cpu, device=device),
|
||||
num_reqs=len(seq_lens_cpu),
|
||||
num_actual_tokens=common_attn_metadata.num_actual_tokens,
|
||||
max_query_len=seqlens_q_local.max(),
|
||||
@@ -808,14 +808,12 @@ def create_fast_prefill_custom_backend(
|
||||
|
||||
|
||||
def compute_causal_conv1d_metadata(
|
||||
query_start_loc_p_cpu: torch.Tensor,
|
||||
*,
|
||||
device: torch.device,
|
||||
):
|
||||
query_start_loc_p_cpu: torch.Tensor, *, device: torch.device
|
||||
) -> tuple[dict[int, dict[str, Any]], torch.Tensor, torch.Tensor]:
|
||||
# Needed for causal_conv1d. Use the CPU query_start_loc to avoid DtoH sync.
|
||||
assert query_start_loc_p_cpu.device.type == "cpu"
|
||||
seqlens = query_start_loc_p_cpu.diff()
|
||||
nums_dict = {} # type: ignore
|
||||
nums_dict: dict[int, dict[str, Any]] = {}
|
||||
batch_ptr = None
|
||||
token_chunk_offset_ptr = None
|
||||
for BLOCK_M in [8]: # cover all BLOCK_M values
|
||||
@@ -823,7 +821,7 @@ def compute_causal_conv1d_metadata(
|
||||
nums_dict[BLOCK_M] = {}
|
||||
nums_dict[BLOCK_M]["nums"] = nums
|
||||
nums_dict[BLOCK_M]["tot"] = nums.sum().item()
|
||||
mlist = torch.from_numpy(np.repeat(np.arange(len(nums)), nums))
|
||||
mlist = np_to_pinned_tensor(np.repeat(np.arange(len(nums)), nums))
|
||||
nums_dict[BLOCK_M]["mlist"] = mlist
|
||||
mlist_len = len(nums_dict[BLOCK_M]["mlist"])
|
||||
nums_dict[BLOCK_M]["mlist_len"] = mlist_len
|
||||
@@ -831,7 +829,7 @@ def compute_causal_conv1d_metadata(
|
||||
offsetlist = [] # type: ignore
|
||||
for idx, num in enumerate(nums):
|
||||
offsetlist.extend(range(num))
|
||||
offsetlist = torch.tensor(offsetlist, dtype=torch.int32)
|
||||
offsetlist = torch.tensor(offsetlist, dtype=torch.int32, pin_memory=PIN_MEMORY)
|
||||
nums_dict[BLOCK_M]["offsetlist"] = offsetlist
|
||||
|
||||
if batch_ptr is None:
|
||||
@@ -845,16 +843,15 @@ def compute_causal_conv1d_metadata(
|
||||
else:
|
||||
if batch_ptr.nelement() < MAX_NUM_PROGRAMS:
|
||||
batch_ptr.resize_(MAX_NUM_PROGRAMS).fill_(PAD_SLOT_ID)
|
||||
token_chunk_offset_ptr.resize_( # type: ignore
|
||||
MAX_NUM_PROGRAMS
|
||||
).fill_(PAD_SLOT_ID)
|
||||
assert token_chunk_offset_ptr is not None
|
||||
token_chunk_offset_ptr.resize_(MAX_NUM_PROGRAMS).fill_(PAD_SLOT_ID)
|
||||
|
||||
assert batch_ptr is not None
|
||||
batch_ptr[0:mlist_len].copy_(mlist, non_blocking=True)
|
||||
token_chunk_offset_ptr[ # type: ignore
|
||||
0:mlist_len
|
||||
].copy_(offsetlist, non_blocking=True)
|
||||
assert token_chunk_offset_ptr is not None
|
||||
token_chunk_offset_ptr[0:mlist_len].copy_(offsetlist, non_blocking=True)
|
||||
nums_dict[BLOCK_M]["batch_ptr"] = batch_ptr
|
||||
nums_dict[BLOCK_M]["token_chunk_offset_ptr"] = token_chunk_offset_ptr # type: ignore
|
||||
nums_dict[BLOCK_M]["token_chunk_offset_ptr"] = token_chunk_offset_ptr
|
||||
|
||||
return nums_dict, batch_ptr, token_chunk_offset_ptr
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.kv_offload.base import (
|
||||
BlockIDsLoadStoreSpec,
|
||||
CanonicalKVCacheRef,
|
||||
@@ -156,7 +156,7 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None:
|
||||
def _new_descriptor_buffers(
|
||||
num_copy_ops: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
pin = is_pin_memory_available()
|
||||
pin = PIN_MEMORY
|
||||
# CUDA cache_kernels.cu requires int64; XPU DMA engine requires uint64.
|
||||
ptr_dtype = torch.uint64 if current_platform.is_xpu() else torch.int64
|
||||
return (
|
||||
@@ -482,7 +482,7 @@ class CpuGpuOffloadingHandlers:
|
||||
num_cpu_blocks: int,
|
||||
mmap_region: SharedOffloadRegion | None = None,
|
||||
):
|
||||
pin_memory = is_pin_memory_available()
|
||||
pin_memory = PIN_MEMORY
|
||||
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
|
||||
self._mmap_region = mmap_region
|
||||
if mmap_region is not None and pin_memory:
|
||||
|
||||
@@ -7,9 +7,7 @@ import torch
|
||||
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
|
||||
pin_memory = is_pin_memory_available()
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -134,7 +132,7 @@ class PoolingMetadata:
|
||||
num_scheduled_tokens_cpu = torch.from_numpy(num_scheduled_tokens_np)
|
||||
if query_start_loc_gpu is None:
|
||||
cumsum = torch.zeros(
|
||||
n_seq + 1, dtype=torch.int64, pin_memory=pin_memory, device="cpu"
|
||||
n_seq + 1, dtype=torch.int64, pin_memory=PIN_MEMORY, device="cpu"
|
||||
)
|
||||
torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:])
|
||||
cumsum = cumsum.to(device, non_blocking=True)
|
||||
|
||||
@@ -7,6 +7,7 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.sample.logits_processor.interface import (
|
||||
BatchUpdate,
|
||||
LogitsProcessor,
|
||||
@@ -118,7 +119,6 @@ class MinPLogitsProcessor(LogitsProcessor):
|
||||
class LogitBiasLogitsProcessor(LogitsProcessor):
|
||||
def __init__(self, _, device: torch.device, is_pin_memory: bool):
|
||||
self.device = device
|
||||
self.pin_memory = is_pin_memory
|
||||
self.biases: dict[int, dict[int, float]] = {}
|
||||
|
||||
self.bias_tensor: torch.Tensor = torch.tensor(())
|
||||
@@ -154,9 +154,7 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
|
||||
)
|
||||
|
||||
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
|
||||
return torch.tensor(
|
||||
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
|
||||
).to(device=self.device, non_blocking=True)
|
||||
return async_tensor_h2d(data, device=self.device, dtype=dtype)
|
||||
|
||||
def apply(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
if self.biases:
|
||||
@@ -170,7 +168,6 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
):
|
||||
# index -> (min_toks, output_token_ids, stop_token_ids)
|
||||
self.device = device
|
||||
self.pin_memory = is_pin_memory
|
||||
self.min_toks: dict[int, tuple[int, Sequence[int], set[int]]] = {}
|
||||
|
||||
# (req_idx_tensor,eos_tok_id_tensor)
|
||||
@@ -227,9 +224,7 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
)
|
||||
|
||||
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
|
||||
return torch.tensor(
|
||||
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
|
||||
).to(device=self.device, non_blocking=True)
|
||||
return async_tensor_h2d(data, device=self.device, dtype=dtype)
|
||||
|
||||
def apply(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
if self.min_toks:
|
||||
@@ -283,8 +278,8 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
toks_arr = np.concatenate(all_toks)
|
||||
# (row_indices, token_indices) for index_put_ to set -inf.
|
||||
logits_slice = (
|
||||
torch.from_numpy(rows_arr).to(self.device, non_blocking=True),
|
||||
torch.from_numpy(toks_arr).to(self.device, non_blocking=True),
|
||||
async_tensor_h2d(rows_arr, device=self.device),
|
||||
async_tensor_h2d(toks_arr, device=self.device),
|
||||
)
|
||||
logits.index_put_(logits_slice, self.neg_inf_tensor)
|
||||
|
||||
|
||||
@@ -4,8 +4,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.utils import apply_penalties
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import make_tensor_with_pad
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, make_tensor_with_pad
|
||||
|
||||
|
||||
def apply_all_penalties(
|
||||
@@ -52,6 +51,6 @@ def _convert_to_tensors(
|
||||
pad=vocab_size,
|
||||
device="cpu",
|
||||
dtype=torch.int64,
|
||||
pin_memory=is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
return output_tokens_tensor.to(device, non_blocking=True)
|
||||
|
||||
@@ -6,7 +6,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.outputs import LogprobsTensors, SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.ops.bad_words import apply_bad_words
|
||||
@@ -65,7 +65,7 @@ class Sampler(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.topk_topp_sampler = TopKTopPSampler(logprobs_mode, use_fp64_gumbel)
|
||||
self.pin_memory = is_pin_memory_available()
|
||||
self.pin_memory = PIN_MEMORY
|
||||
self.logprobs_mode = logprobs_mode
|
||||
self.use_fp64_gumbel = use_fp64_gumbel
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.sample.logits_processor.interface import (
|
||||
BatchUpdate,
|
||||
MoveDirectionality,
|
||||
@@ -22,12 +22,11 @@ def maybe_create_thinking_budget_state_holder(
|
||||
max_num_seqs: int,
|
||||
num_spec_tokens: int,
|
||||
device: torch.device,
|
||||
is_pin_memory: bool,
|
||||
) -> "ThinkingBudgetStateHolder | None":
|
||||
if reasoning_config is None:
|
||||
return None
|
||||
return ThinkingBudgetStateHolder(
|
||||
reasoning_config, max_num_seqs, num_spec_tokens, device, is_pin_memory
|
||||
reasoning_config, max_num_seqs, num_spec_tokens, device, PIN_MEMORY
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ from vllm.multimodal.inputs import (
|
||||
MultiModalSharedField,
|
||||
NestedTensors,
|
||||
)
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.utils import tensor_data
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -327,7 +327,7 @@ class MsgpackDecoder:
|
||||
oob_tensor_provider: OOBTensorProvider | None = None,
|
||||
):
|
||||
self.share_mem = share_mem
|
||||
self.pin_tensors = is_pin_memory_available()
|
||||
self.pin_tensors = PIN_MEMORY
|
||||
args = () if t is None else (t,)
|
||||
self.decoder = msgpack.Decoder(
|
||||
*args, ext_hook=self.ext_hook, dec_hook=self.dec_hook
|
||||
|
||||
@@ -8,7 +8,7 @@ import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.simple_kv_offload.copy_backend import DmaCopyBackend
|
||||
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
@@ -149,7 +149,7 @@ class SimpleCPUOffloadWorker:
|
||||
(self.num_cpu_blocks * total_bytes_per_block) / (1024**3),
|
||||
)
|
||||
|
||||
pin_memory = is_pin_memory_available()
|
||||
pin_memory = PIN_MEMORY
|
||||
if not pin_memory:
|
||||
logger.warning(
|
||||
"Pinned memory not available. CPU offload performance may be degraded."
|
||||
|
||||
@@ -12,7 +12,7 @@ from vllm.config import CUDAGraphMode, VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.attention.backend import AttentionMetadataBuilder, CommonAttentionMetadata
|
||||
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
|
||||
from vllm.v1.utils import CpuGpuBuffer
|
||||
@@ -58,7 +58,7 @@ class ExtractHiddenStatesProposer:
|
||||
self.backup_next_token_ids = CpuGpuBuffer(
|
||||
max_batch_size,
|
||||
dtype=torch.int32,
|
||||
pin_memory=is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
device=device,
|
||||
with_numpy=True,
|
||||
)
|
||||
@@ -317,7 +317,6 @@ class ExtractHiddenStatesProposer:
|
||||
(batch_size, 1). For each request we either use the sampled token
|
||||
(if valid and not discarded) or a backup token from the request state.
|
||||
"""
|
||||
num_reqs = gpu_input_batch.num_reqs
|
||||
|
||||
# Precompute backup token IDs for discarded requests.
|
||||
num_reqs = gpu_input_batch.num_reqs
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
|
||||
from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadata
|
||||
@@ -228,7 +228,7 @@ class SpecDecodeBaseProposer:
|
||||
self.backup_next_token_ids = CpuGpuBuffer(
|
||||
self.max_batch_size,
|
||||
dtype=torch.int32,
|
||||
pin_memory=is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
device=device,
|
||||
with_numpy=True,
|
||||
)
|
||||
@@ -239,9 +239,7 @@ class SpecDecodeBaseProposer:
|
||||
self._last_draft_probs: torch.Tensor | None = None
|
||||
|
||||
self._slot_mapping_buffer = torch.zeros(
|
||||
self.max_positions,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
self.max_positions, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
# Determine allowed attention backends once during initialization.
|
||||
@@ -1127,7 +1125,7 @@ class SpecDecodeBaseProposer:
|
||||
new_query_start_loc_cpu = torch.zeros(
|
||||
query_start_loc_cpu.shape,
|
||||
dtype=torch.int32,
|
||||
pin_memory=is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
new_query_start_loc_np = new_query_start_loc_cpu.numpy()
|
||||
np.cumsum(new_num_tokens_per_req_np, out=new_query_start_loc_np[1:])
|
||||
@@ -1160,11 +1158,11 @@ class SpecDecodeBaseProposer:
|
||||
# q1 + 0, q1 + 1, q1 + 2, q1 + 3, // req 2
|
||||
# q1 + q2 + 0, q1 + q2 + 1, q1 + q2 + 2] // req 3
|
||||
token_indices_np = token_offsets + old_query_start_locs_expanded
|
||||
token_indices = torch.from_numpy(token_indices_np).to(device, non_blocking=True)
|
||||
token_indices = async_tensor_h2d(token_indices_np, device=device)
|
||||
|
||||
spec_common_attn_metadata = CommonAttentionMetadata(
|
||||
query_start_loc=new_query_start_loc_cpu.to(device, non_blocking=True),
|
||||
seq_lens=new_seq_lens_cpu.to(device, non_blocking=True),
|
||||
query_start_loc=async_tensor_h2d(new_query_start_loc_cpu, device=device),
|
||||
seq_lens=async_tensor_h2d(new_seq_lens_cpu, device=device),
|
||||
query_start_loc_cpu=new_query_start_loc_cpu,
|
||||
_seq_lens_cpu=new_seq_lens_cpu,
|
||||
_num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu,
|
||||
|
||||
@@ -545,7 +545,7 @@ def update_ngram_gpu_tensors_incremental(
|
||||
num_tokens = input_batch.num_tokens_no_spec[idx]
|
||||
if num_tokens > 0:
|
||||
token_ids_gpu_tensor[idx, :num_tokens].copy_(
|
||||
input_batch.token_ids_cpu_tensor[idx, :num_tokens],
|
||||
input_batch.token_ids_cpu_tensor[idx, :num_tokens].pin_memory(),
|
||||
non_blocking=True,
|
||||
)
|
||||
|
||||
@@ -591,7 +591,7 @@ def update_ngram_gpu_tensors_incremental(
|
||||
num_tokens = input_batch.num_tokens_no_spec[new_req_idx]
|
||||
if num_tokens > 0:
|
||||
token_ids_gpu_tensor[new_req_idx, :num_tokens].copy_(
|
||||
input_batch.token_ids_cpu_tensor[new_req_idx, :num_tokens],
|
||||
input_batch.token_ids_cpu_tensor[new_req_idx, :num_tokens].pin_memory(),
|
||||
non_blocking=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
@@ -139,7 +139,7 @@ class LMFormatEnforcerBackend(StructuredOutputBackend):
|
||||
(max_num_seqs, (self.vocab_size + 31) // 32),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
pin_memory=is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
|
||||
def destroy(self):
|
||||
|
||||
@@ -15,7 +15,7 @@ from regex import escape as regex_escape
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
@@ -101,7 +101,7 @@ class OutlinesBackend(StructuredOutputBackend):
|
||||
(max_num_seqs, (self.vocab_size + 31) // 32),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
pin_memory=is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
|
||||
def destroy(self):
|
||||
|
||||
@@ -10,7 +10,6 @@ from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import regex as re
|
||||
import torch
|
||||
from cachetools import LRUCache
|
||||
@@ -18,7 +17,7 @@ from cachetools import LRUCache
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -123,11 +122,13 @@ def apply_grammar_bitmask(
|
||||
out_indices = []
|
||||
|
||||
# Reorder the bitmask to match the order of the requests in the batch.
|
||||
sorted_bitmask = np.full(
|
||||
shape=(logits.shape[0], grammar_bitmask.shape[1]),
|
||||
fill_value=-1,
|
||||
dtype=grammar_bitmask.dtype,
|
||||
sorted_bitmask_tensor = torch.full(
|
||||
(logits.shape[0], grammar_bitmask.shape[1]),
|
||||
-1,
|
||||
dtype=torch.from_numpy(grammar_bitmask[:0]).dtype,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
sorted_bitmask = sorted_bitmask_tensor.numpy()
|
||||
cumulative_index = 0
|
||||
for req_id in grammar_output.structured_output_request_ids:
|
||||
num_spec_tokens = len(spec_tokens.get(req_id, ()))
|
||||
@@ -138,10 +139,8 @@ def apply_grammar_bitmask(
|
||||
out_indices.append(bitmask_index)
|
||||
cumulative_index += 1 + num_spec_tokens
|
||||
|
||||
# Copy async to device as tensor.
|
||||
grammar_bitmask = torch.from_numpy(sorted_bitmask).to(
|
||||
logits.device, non_blocking=True
|
||||
)
|
||||
# Copy async to device.
|
||||
grammar_bitmask = sorted_bitmask_tensor.to(logits.device, non_blocking=True)
|
||||
|
||||
# If the length of out indices and the logits have the same shape
|
||||
# we don't need to pass indices to the kernel,
|
||||
@@ -154,11 +153,9 @@ def apply_grammar_bitmask(
|
||||
# xgrammar expects a python list of indices but it will actually work with
|
||||
# a tensor. If we copy the tensor ourselves here we can do it in a
|
||||
# non_blocking manner and there should be no cpu sync within xgrammar.
|
||||
pin_memory = is_pin_memory_available()
|
||||
index_tensor = torch.tensor(
|
||||
out_indices, dtype=torch.int32, device="cpu", pin_memory=pin_memory
|
||||
index_tensor = async_tensor_h2d(
|
||||
out_indices, dtype=torch.int32, device=logits.device
|
||||
)
|
||||
index_tensor = index_tensor.to(logits.device, non_blocking=True)
|
||||
|
||||
xgr.apply_token_bitmask_inplace(logits, grammar_bitmask, indices=index_tensor)
|
||||
return
|
||||
|
||||
+2
-1
@@ -31,6 +31,7 @@ from vllm.logger import init_logger
|
||||
from vllm.usage.usage_lib import UsageContext, is_usage_stats_enabled, usage_message
|
||||
from vllm.utils.network_utils import get_open_zmq_ipc_path, get_tcp_uri
|
||||
from vllm.utils.system_utils import decorate_logs, kill_process_tree, set_process_title
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -114,7 +115,7 @@ class CpuGpuBuffer:
|
||||
*size: int | torch.SymInt,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
pin_memory: bool,
|
||||
pin_memory: bool = PIN_MEMORY,
|
||||
with_numpy: bool = True,
|
||||
) -> None:
|
||||
# these buffers are mutable runtime state, so allocate them as normal
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
# Patch torch APIs
|
||||
import torch
|
||||
|
||||
@@ -45,11 +47,14 @@ import vllm.utils.torch_utils as torch_utils
|
||||
|
||||
|
||||
def async_tensor_h2d(
|
||||
data: list,
|
||||
dtype: torch.dtype,
|
||||
data: list | np.ndarray | torch.Tensor,
|
||||
device: str | torch.device,
|
||||
pin_memory: bool = False,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
if isinstance(data, np.ndarray):
|
||||
data = torch.from_numpy(data)
|
||||
if isinstance(data, torch.Tensor):
|
||||
return data.to(dtype=dtype)
|
||||
return torch.tensor(data, dtype=dtype, device="cpu")
|
||||
|
||||
|
||||
|
||||
@@ -36,10 +36,9 @@ def async_copy_to_gpu(
|
||||
assert device is not None
|
||||
out = torch.empty_like(x, device=device)
|
||||
|
||||
# Copy directly to GPU — explicit pin_memory() causes sporadic stalls
|
||||
# under high concurrency due to CUDA driver contention. The driver
|
||||
# handles the transfer efficiently without manual pinning.
|
||||
return out.copy_(x, non_blocking=True)
|
||||
# pin_memory() is no-op if the memory is already pinned.
|
||||
pinned = x.pin_memory()
|
||||
return out.copy_(pinned, non_blocking=True)
|
||||
|
||||
|
||||
class UvaBuffer:
|
||||
@@ -183,7 +182,7 @@ class StagedWriteTensor:
|
||||
|
||||
# Special handling for write_contents
|
||||
write_contents = async_tensor_h2d(
|
||||
self._staged_write_contents, self.dtype, self.device
|
||||
self._staged_write_contents, device=self.device, dtype=self.dtype
|
||||
)
|
||||
|
||||
# Write diffs to the GPU buffer
|
||||
@@ -255,7 +254,7 @@ class FusedStagedWriter:
|
||||
indices_uva = self.indices.copy_to_uva(indices)
|
||||
starts_uva = self.starts.copy_to_uva(starts)
|
||||
cu_lens_uva = self.cu_lens.copy_to_uva(cu_lens)
|
||||
contents_gpu = async_tensor_h2d(contents, torch.int32, self.device)
|
||||
contents_gpu = async_tensor_h2d(contents, device=self.device, dtype=torch.int32)
|
||||
|
||||
_apply_write_kernel[(len(group_ids),)](
|
||||
output_ptrs,
|
||||
|
||||
@@ -49,12 +49,11 @@ class EncoderRunner:
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_mm_encoder(
|
||||
self,
|
||||
mm_kwargs: list[tuple[str, MultiModalKwargsItem]],
|
||||
self, mm_kwargs: list[tuple[str, MultiModalKwargsItem]]
|
||||
) -> list[torch.Tensor]:
|
||||
encoder_outputs: list[torch.Tensor] = []
|
||||
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
mm_kwargs, device=self.device, pin_memory=False
|
||||
mm_kwargs, device=self.device, pin_memory=True
|
||||
):
|
||||
batch_outputs = self.model.embed_multimodal(**mm_kwargs_batch)
|
||||
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
|
||||
|
||||
@@ -46,8 +46,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
|
||||
@@ -498,7 +497,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
|
||||
self.kv_block_zeroer = KVBlockZeroer(
|
||||
self.device,
|
||||
is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
attn_groups_iter=(g for groups in self.attn_groups for g in groups),
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
cache_dtype=self.cache_config.cache_dtype,
|
||||
|
||||
@@ -170,7 +170,8 @@ class WhisperModelState(ModelState):
|
||||
for_capture: bool,
|
||||
num_reqs: int,
|
||||
) -> dict[int, tuple[torch.Tensor, np.ndarray]]:
|
||||
encoder_seq_lens_np = np.zeros(num_reqs, dtype=np.int32)
|
||||
encoder_seq_lens = torch.zeros(num_reqs, dtype=torch.int32, pin_memory=True)
|
||||
encoder_seq_lens_np = encoder_seq_lens.numpy()
|
||||
if not for_capture:
|
||||
# During normal execution, use actual encoder lengths.
|
||||
for i, req_id in enumerate(req_ids):
|
||||
@@ -183,9 +184,7 @@ class WhisperModelState(ModelState):
|
||||
# is captured with the correct value for cross-attention.
|
||||
encoder_seq_lens_np[:] = self.max_encoder_len
|
||||
|
||||
self.encoder_seq_lens_gpu[:num_reqs].copy_(
|
||||
torch.from_numpy(encoder_seq_lens_np), non_blocking=True
|
||||
)
|
||||
self.encoder_seq_lens_gpu[:num_reqs].copy_(encoder_seq_lens, non_blocking=True)
|
||||
self.encoder_seq_lens_gpu[num_reqs:].fill_(0)
|
||||
encoder_seq_lens_gpu = self.encoder_seq_lens_gpu[:num_reqs]
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams, SamplingType
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.utils.collection_utils import swap_dict_values
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
@@ -95,7 +96,6 @@ class InputBatch:
|
||||
max_model_len: int,
|
||||
max_num_batched_tokens: int,
|
||||
device: torch.device,
|
||||
pin_memory: bool,
|
||||
vocab_size: int,
|
||||
block_sizes: list[int], # The block_size of each kv cache group
|
||||
kernel_block_sizes: list[int],
|
||||
@@ -112,7 +112,6 @@ class InputBatch:
|
||||
max_num_reqs,
|
||||
num_spec_tokens,
|
||||
device,
|
||||
pin_memory,
|
||||
)
|
||||
self.thinking_token_budget_reqs: set[str] = set()
|
||||
self.is_pooling_model = is_pooling_model
|
||||
@@ -120,7 +119,6 @@ class InputBatch:
|
||||
self.max_model_len = max_model_len
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
self.device = device
|
||||
self.pin_memory = pin_memory
|
||||
self.vocab_size = vocab_size
|
||||
|
||||
self._req_ids: list[str | None] = []
|
||||
@@ -138,7 +136,10 @@ class InputBatch:
|
||||
)
|
||||
self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
|
||||
self.is_token_ids_tensor = torch.zeros(
|
||||
(max_num_reqs, max_model_len), device="cpu", dtype=bool, pin_memory=False
|
||||
(max_num_reqs, max_model_len),
|
||||
device="cpu",
|
||||
dtype=bool,
|
||||
pin_memory=False,
|
||||
)
|
||||
self.is_token_ids = self.is_token_ids_tensor.numpy()
|
||||
# Store prompt embeddings per request to avoid OOM from large upfront
|
||||
@@ -149,21 +150,21 @@ class InputBatch:
|
||||
(max_num_reqs,),
|
||||
device="cpu",
|
||||
dtype=torch.int32,
|
||||
pin_memory=pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
self.num_tokens_no_spec = self.num_tokens_no_spec_cpu_tensor.numpy()
|
||||
self.num_prompt_tokens_cpu_tensor = torch.zeros(
|
||||
(max_num_reqs,),
|
||||
device="cpu",
|
||||
dtype=torch.int32,
|
||||
pin_memory=pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
self.num_prompt_tokens = self.num_prompt_tokens_cpu_tensor.numpy()
|
||||
self.num_computed_tokens_cpu_tensor = torch.zeros(
|
||||
(max_num_reqs,),
|
||||
device="cpu",
|
||||
dtype=torch.int32,
|
||||
pin_memory=pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
self.num_computed_tokens_cpu = self.num_computed_tokens_cpu_tensor.numpy()
|
||||
|
||||
@@ -172,7 +173,7 @@ class InputBatch:
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_model_len=max_model_len,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
pin_memory=pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
device=device,
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
@@ -185,7 +186,7 @@ class InputBatch:
|
||||
(max_num_reqs,), dtype=torch.float32, device=device
|
||||
)
|
||||
self.temperature_cpu_tensor = torch.empty(
|
||||
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory
|
||||
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
self.temperature_cpu = self.temperature_cpu_tensor.numpy()
|
||||
self.greedy_reqs: set[str] = set()
|
||||
@@ -193,14 +194,14 @@ class InputBatch:
|
||||
|
||||
self.top_p = torch.empty((max_num_reqs,), dtype=torch.float32, device=device)
|
||||
self.top_p_cpu_tensor = torch.empty(
|
||||
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory
|
||||
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
self.top_p_cpu = self.top_p_cpu_tensor.numpy()
|
||||
self.top_p_reqs: set[str] = set()
|
||||
|
||||
self.top_k = torch.empty((max_num_reqs,), dtype=torch.int32, device=device)
|
||||
self.top_k_cpu_tensor = torch.empty(
|
||||
(max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=pin_memory
|
||||
(max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
self.top_k_cpu = self.top_k_cpu_tensor.numpy()
|
||||
self.top_k_reqs: set[str] = set()
|
||||
@@ -210,7 +211,7 @@ class InputBatch:
|
||||
(max_num_reqs,), dtype=torch.float, device=device
|
||||
)
|
||||
self.frequency_penalties_cpu_tensor = torch.empty(
|
||||
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory
|
||||
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
self.frequency_penalties_cpu = self.frequency_penalties_cpu_tensor.numpy()
|
||||
self.frequency_penalties_reqs: set[str] = set()
|
||||
@@ -220,7 +221,7 @@ class InputBatch:
|
||||
(max_num_reqs,), dtype=torch.float, device=device
|
||||
)
|
||||
self.presence_penalties_cpu_tensor = torch.empty(
|
||||
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory
|
||||
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
self.presence_penalties_cpu = self.presence_penalties_cpu_tensor.numpy()
|
||||
self.presence_penalties_reqs: set[str] = set()
|
||||
@@ -230,14 +231,14 @@ class InputBatch:
|
||||
(max_num_reqs,), dtype=torch.float, device=device
|
||||
)
|
||||
self.repetition_penalties_cpu_tensor = torch.empty(
|
||||
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory
|
||||
(max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
self.repetition_penalties_cpu = self.repetition_penalties_cpu_tensor.numpy()
|
||||
self.repetition_penalties_reqs: set[str] = set()
|
||||
|
||||
# Speculative decoding
|
||||
self.num_accepted_tokens_cpu_tensor = torch.ones(
|
||||
(max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=pin_memory
|
||||
(max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
self.num_accepted_tokens_cpu = self.num_accepted_tokens_cpu_tensor.numpy()
|
||||
|
||||
@@ -963,7 +964,7 @@ class InputBatch:
|
||||
(self.num_reqs, max_prompt_len),
|
||||
device="cpu",
|
||||
dtype=torch.int64,
|
||||
pin_memory=self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
prompt_token_ids = prompt_token_ids_cpu_tensor.numpy()
|
||||
prompt_token_ids[:] = self.token_ids_cpu[:num_reqs, :max_prompt_len]
|
||||
|
||||
@@ -116,8 +116,10 @@ from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.nvtx_pytorch_hooks import PytHooks
|
||||
from vllm.utils.platform_utils import is_pin_memory_available, num_compute_units
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
async_tensor_h2d,
|
||||
get_dtype_size,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
@@ -441,7 +443,6 @@ class GPUModelRunner(
|
||||
scheduler_config = self.scheduler_config
|
||||
parallel_config = self.parallel_config
|
||||
self.device = device
|
||||
self.pin_memory = is_pin_memory_available()
|
||||
self.dtype = self.model_config.dtype
|
||||
|
||||
self.kv_cache_dtype = kv_cache_dtype_str_to_dtype(
|
||||
@@ -666,7 +667,6 @@ class GPUModelRunner(
|
||||
max_model_len=max(self.max_model_len, self.max_encoder_len),
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=[placeholder_block_size],
|
||||
kernel_block_sizes=[placeholder_block_size],
|
||||
@@ -674,7 +674,7 @@ class GPUModelRunner(
|
||||
logitsprocs=build_logitsprocs(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
self.pin_memory,
|
||||
PIN_MEMORY,
|
||||
self.is_pooling_model,
|
||||
custom_logitsprocs,
|
||||
),
|
||||
@@ -729,7 +729,7 @@ class GPUModelRunner(
|
||||
self.max_num_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.optimistic_seq_lens_cpu = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, pin_memory=self.pin_memory
|
||||
self.max_num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY
|
||||
)
|
||||
self.num_computed_tokens = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=self.device
|
||||
@@ -846,7 +846,7 @@ class GPUModelRunner(
|
||||
and self.speculative_config.use_ngram_gpu()
|
||||
):
|
||||
self._num_valid_draft_tokens_cpu = torch.empty(
|
||||
self.max_num_reqs, dtype=torch.int32, pin_memory=self.pin_memory
|
||||
self.max_num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY
|
||||
)
|
||||
self._num_valid_draft_tokens_event = torch.cuda.Event()
|
||||
self._num_valid_draft_tokens_copy_stream = torch.cuda.Stream()
|
||||
@@ -857,7 +857,7 @@ class GPUModelRunner(
|
||||
(self.max_num_reqs, 1),
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
pin_memory=self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
|
||||
# Pre-allocated tensor for copying valid sampled token counts to CPU,
|
||||
@@ -879,7 +879,7 @@ class GPUModelRunner(
|
||||
(self.max_num_reqs, self.num_spec_tokens),
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
pin_memory=self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
if self.use_async_scheduling:
|
||||
self.valid_sampled_token_count_event = torch.Event()
|
||||
@@ -888,7 +888,7 @@ class GPUModelRunner(
|
||||
self.max_num_reqs,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
pin_memory=self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
|
||||
# Model weight offloader
|
||||
@@ -1000,7 +1000,6 @@ class GPUModelRunner(
|
||||
*size,
|
||||
dtype=dtype,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
with_numpy=numpy,
|
||||
)
|
||||
|
||||
@@ -1055,7 +1054,7 @@ class GPUModelRunner(
|
||||
token_type_ids.append(ids)
|
||||
|
||||
token_type_ids_cpu = torch.empty(
|
||||
sum(seq_lens_cpu), dtype=torch.int32, pin_memory=self.pin_memory
|
||||
sum(seq_lens_cpu), dtype=torch.int32, pin_memory=PIN_MEMORY
|
||||
)
|
||||
torch.cat(token_type_ids, out=token_type_ids_cpu)
|
||||
model_kwargs["token_type_ids"] = token_type_ids_cpu.to(
|
||||
@@ -1095,12 +1094,12 @@ class GPUModelRunner(
|
||||
"""
|
||||
self._kv_block_zeroer = KVBlockZeroer(
|
||||
self.device,
|
||||
self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
attn_groups_iter=self._kv_cache_spec_attn_group_iterator(),
|
||||
kernel_block_sizes=self._kernel_block_sizes,
|
||||
cache_dtype=self.cache_config.cache_dtype,
|
||||
runner_only_attn_layers=self.runner_only_attn_layers,
|
||||
static_forward_context=(self.compilation_config.static_forward_context),
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
)
|
||||
|
||||
def _zero_block_ids(self, block_ids: list[int]) -> None:
|
||||
@@ -1651,7 +1650,7 @@ class GPUModelRunner(
|
||||
for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
mm_kwargs,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
):
|
||||
mm_kwargs_combined.update(mm_kwargs_batch)
|
||||
|
||||
@@ -1807,10 +1806,10 @@ class GPUModelRunner(
|
||||
return
|
||||
# Upload the index tensors asynchronously so the scatter can be non-blocking.
|
||||
sampled_tokens_index_tensor = torch.tensor(
|
||||
sample_flattened_indices, dtype=torch.int64, pin_memory=self.pin_memory
|
||||
sample_flattened_indices, dtype=torch.int64, pin_memory=PIN_MEMORY
|
||||
).to(self.device, non_blocking=True)
|
||||
prev_common_req_indices_tensor = torch.tensor(
|
||||
prev_indices, dtype=torch.int64, pin_memory=self.pin_memory
|
||||
prev_indices, dtype=torch.int64, pin_memory=PIN_MEMORY
|
||||
).to(self.device, non_blocking=True)
|
||||
self.input_ids.gpu.scatter_(
|
||||
dim=0,
|
||||
@@ -1826,10 +1825,10 @@ class GPUModelRunner(
|
||||
|
||||
assert isinstance(self._draft_token_ids, torch.Tensor)
|
||||
draft_tokens_index_tensor = torch.tensor(
|
||||
spec_flattened_indices, dtype=torch.int64, pin_memory=self.pin_memory
|
||||
spec_flattened_indices, dtype=torch.int64, pin_memory=PIN_MEMORY
|
||||
).to(self.device, non_blocking=True)
|
||||
prev_draft_token_indices_tensor = torch.tensor(
|
||||
prev_draft_token_indices, dtype=torch.int64, pin_memory=self.pin_memory
|
||||
prev_draft_token_indices, dtype=torch.int64, pin_memory=PIN_MEMORY
|
||||
).to(self.device, non_blocking=True)
|
||||
|
||||
# because input_ids dtype is torch.int32,
|
||||
@@ -2788,21 +2787,16 @@ class GPUModelRunner(
|
||||
# [0, 1, 2, 5, 6, 9]
|
||||
target_logits_indices += self._arange_scratch[: cu_num_draft_tokens[-1]]
|
||||
|
||||
# TODO: Optimize the CPU -> GPU copy.
|
||||
cu_num_draft_tokens = torch.from_numpy(cu_num_draft_tokens).to(
|
||||
self.device, non_blocking=True
|
||||
cu_num_draft_tokens = async_tensor_h2d(cu_num_draft_tokens, device=self.device)
|
||||
cu_num_sampled_tokens = async_tensor_h2d(
|
||||
cu_num_sampled_tokens, device=self.device
|
||||
)
|
||||
cu_num_sampled_tokens = torch.from_numpy(cu_num_sampled_tokens).to(
|
||||
self.device, non_blocking=True
|
||||
logits_indices = async_tensor_h2d(logits_indices, device=self.device)
|
||||
target_logits_indices = async_tensor_h2d(
|
||||
target_logits_indices, device=self.device
|
||||
)
|
||||
logits_indices = torch.from_numpy(logits_indices).to(
|
||||
self.device, non_blocking=True
|
||||
)
|
||||
target_logits_indices = torch.from_numpy(target_logits_indices).to(
|
||||
self.device, non_blocking=True
|
||||
)
|
||||
bonus_logits_indices = torch.from_numpy(bonus_logits_indices).to(
|
||||
self.device, non_blocking=True
|
||||
bonus_logits_indices = async_tensor_h2d(
|
||||
bonus_logits_indices, device=self.device
|
||||
)
|
||||
|
||||
# Compute the draft token ids.
|
||||
@@ -3012,9 +3006,7 @@ class GPUModelRunner(
|
||||
# Track the current index in mm_kwargs/mm_lora_refs to map groups to request IDs
|
||||
current_item_idx = 0
|
||||
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
mm_kwargs,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
mm_kwargs, device=self.device, pin_memory=PIN_MEMORY
|
||||
):
|
||||
batch_outputs: MultiModalEmbeddings
|
||||
|
||||
@@ -3048,7 +3040,7 @@ class GPUModelRunner(
|
||||
group_and_batch_mm_kwargs(
|
||||
[video_mm_kwargs_item],
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3107,7 +3099,10 @@ class GPUModelRunner(
|
||||
|
||||
mm_embeds = list[torch.Tensor]()
|
||||
is_mm_embed = torch.zeros(
|
||||
total_num_scheduled_tokens, dtype=torch.bool, device="cpu"
|
||||
total_num_scheduled_tokens,
|
||||
dtype=torch.bool,
|
||||
device="cpu",
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
|
||||
req_start_idx = 0
|
||||
@@ -3515,8 +3510,7 @@ class GPUModelRunner(
|
||||
token_ids_idx_np = np.nonzero(is_token_ids)[0]
|
||||
# Some tokens ids may need to become embeds
|
||||
if token_ids_idx_np.size > 0:
|
||||
token_ids_idx = torch.from_numpy(token_ids_idx_np)
|
||||
token_ids_idx = token_ids_idx.to(self.device, non_blocking=True)
|
||||
token_ids_idx = async_tensor_h2d(token_ids_idx_np, device=self.device)
|
||||
token_ids = self.input_ids.gpu[token_ids_idx]
|
||||
tokens_to_embeds = self.model.embed_input_ids(input_ids=token_ids)
|
||||
self.inputs_embeds.gpu[token_ids_idx] = tokens_to_embeds
|
||||
@@ -4953,7 +4947,7 @@ class GPUModelRunner(
|
||||
):
|
||||
indices.append(offset + len(tokens) - 1)
|
||||
offset += num_draft + 1
|
||||
indices = torch.tensor(indices, device=self.device)
|
||||
indices = async_tensor_h2d(indices, device=self.device)
|
||||
hidden_states = sample_hidden_states[indices]
|
||||
|
||||
draft_token_ids = self.drafter.propose(
|
||||
@@ -5483,8 +5477,8 @@ class GPUModelRunner(
|
||||
continue
|
||||
|
||||
num_prompt_tokens = len(request.prompt_token_ids)
|
||||
prompt_token_ids = torch.tensor(request.prompt_token_ids).to(
|
||||
self.device, non_blocking=True
|
||||
prompt_token_ids = async_tensor_h2d(
|
||||
request.prompt_token_ids, device=self.device
|
||||
)
|
||||
|
||||
# Set up target LogprobsTensors object.
|
||||
@@ -5651,7 +5645,7 @@ class GPUModelRunner(
|
||||
for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
[(modality, dummy_mm_item)] * max_items_per_batch,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -6995,7 +6989,6 @@ class GPUModelRunner(
|
||||
max_model_len=max_model_len,
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
@@ -7430,7 +7423,7 @@ class GPUModelRunner(
|
||||
self.routed_experts_capturer.device_buffer.shape,
|
||||
dtype=self.routed_experts_capturer.device_buffer.dtype,
|
||||
device="cpu",
|
||||
pin_memory=self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
# ``slot_mapping`` dtype is fixed to int64 by
|
||||
# ``block_table.slot_mapping``; we mirror that here.
|
||||
@@ -7439,7 +7432,7 @@ class GPUModelRunner(
|
||||
(max_tokens,),
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
pin_memory=self.pin_memory,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
# Private device buffer so the shared ``block_table.slot_mapping``
|
||||
# can be overwritten by the next ``_prepare_inputs`` while the
|
||||
|
||||
Reference in New Issue
Block a user