[Perf] Avoid h2d copies from non-pinned CPU tensors (#54299)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-29 12:06:30 +08:00
committed by GitHub
parent 5182f2705b
commit 6c18a54648
14 changed files with 81 additions and 84 deletions
@@ -11,6 +11,7 @@ from vllm.config import get_current_vllm_config
from vllm.config.pooler import TokenPoolingType
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.pool.metadata import PoolingMetadata
TokenPoolingMethodOutputItem: TypeAlias = torch.Tensor | None
@@ -131,7 +132,7 @@ class StepPool(AllPool):
if step_tag_id is not None:
idx_cpu = (token_id_cpu == step_tag_id).nonzero(as_tuple=True)[0]
idx = idx_cpu.to(data.device, non_blocking=True)
idx = async_tensor_h2d(idx_cpu, data.device)
data = data[idx]
pooled_data.append(data)
+11 -10
View File
@@ -158,13 +158,13 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
yield module
return
original_device_states: dict[str, torch.device] = {}
cpu_params: set[str] = set()
uva_offloaded_parameters: list[str] = []
# Store original device states and move parameters to GPU if they're on CPU
# Store which parameters are on CPU and move them to the GPU
for name, p in module.named_parameters():
if p.device.type == "cpu":
original_device_states[name] = p.device
cpu_params.add(name)
p.data = p.data.to(target_device)
if getattr(p, "_vllm_is_uva_offloaded", False):
uva_offloaded_parameters.append(name)
@@ -178,20 +178,21 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
is_pin_memory_available()
and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY
)
# Restore parameters to their original devices, ignoring new parameters
# Restore the CPU-resident parameters, ignoring new parameters.
for name, p in module.named_parameters():
if name in original_device_states:
original_device: torch.device = original_device_states[name]
p.data = p.data.to(original_device)
if name in cpu_params:
p.data = torch.empty_like(
p.data, device="cpu", pin_memory=use_pin_memory
).copy_(p.data)
# parameter is UVA offloaded, but was replaced with a new device tensor
# re-offload it to CPU using UVA
if name in uva_offloaded_parameters and not getattr(
p, "_vllm_is_uva_offloaded", False
):
cpu_data = p.data.to(device="cpu")
if use_pin_memory:
cpu_data = cpu_data.pin_memory()
cpu_data = torch.empty_like(
p.data, device="cpu", pin_memory=use_pin_memory
).copy_(p.data)
p.data = get_accelerator_view_from_cpu_tensor(cpu_data)
p._vllm_is_uva_offloaded = True
+3 -5
View File
@@ -425,10 +425,11 @@ class Ernie4_5_VisionTransformer(nn.Module):
return self.patch_embed.proj.weight.device
def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
device = self.device
pos_ids = []
for t, h, w in grid_thw:
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w)
wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1)
hpos_ids = (
hpos_ids.reshape(
h // self.spatial_merge_size,
@@ -453,9 +454,6 @@ class Ernie4_5_VisionTransformer(nn.Module):
pos_ids = torch.cat(pos_ids, dim=0)
max_grid_size = grid_thw[:, 1:].max()
rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
# `pos_ids` is built on the host; stage it over non-blocking so the
# gather below doesn't index a device tensor with a CPU one.
pos_ids = pos_ids.to(rotary_pos_emb_full.device, non_blocking=True)
rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
return rotary_pos_emb
+3 -3
View File
@@ -709,10 +709,11 @@ class Glm4vVisionTransformer(nn.Module):
def rot_pos_emb(
self, grid_thw: list[list[int]]
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
device = self.device
pos_ids = []
for t, h, w in grid_thw:
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w)
wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1)
hpos_ids = (
hpos_ids.reshape(
h // self.spatial_merge_size,
@@ -740,7 +741,6 @@ class Glm4vVisionTransformer(nn.Module):
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
pos_ids = pos_ids.to(cos.device, non_blocking=True)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
return cos_combined, sin_combined, pos_ids
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import PIN_MEMORY
from .utils import AutoWeightsLoader, WeightsMapper
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
@@ -85,7 +86,9 @@ class Idefics2VisionEmbeddings(nn.Module):
1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side
)
position_ids = torch.full(
size=(batch_size, max_nb_patches_h * max_nb_patches_w), fill_value=0
size=(batch_size, max_nb_patches_h * max_nb_patches_w),
fill_value=0,
pin_memory=PIN_MEMORY,
)
with gpu_sync_allowed():
+3 -6
View File
@@ -32,6 +32,7 @@ from vllm.model_executor.models.phi4mm_utils import (
get_offset,
unfold_tensor,
)
from vllm.utils.torch_utils import async_tensor_h2d
class ConformerEncoderLayer(nn.Module):
@@ -586,9 +587,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
seq_len, batch_size, self.chunk_size, self.left_chunk
)
device = xs_pad.device
enc_streaming_mask = enc_streaming_mask.contiguous().to(
device, non_blocking=True
)
enc_streaming_mask = async_tensor_h2d(enc_streaming_mask.contiguous(), device)
xs_pad = xs_pad.to(device)
input_tensor = xs_pad
@@ -921,9 +920,7 @@ class ConformerEncoder(TransformerEncoderBase):
enc_streaming_mask = self._streaming_mask(
max_audio_length, batch_size, self.chunk_size, self.left_chunk
)
enc_streaming_mask = enc_streaming_mask.contiguous().to(
device, non_blocking=True
)
enc_streaming_mask = async_tensor_h2d(enc_streaming_mask.contiguous(), device)
if mask is None:
return enc_streaming_mask
+5 -7
View File
@@ -715,8 +715,11 @@ class Qwen2_5_VisionTransformer(nn.Module):
return self.patch_embed.proj.weight.device
def rotary_pos_emb_thw(self, t, h, w):
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max(h, w))
hpos_ids = torch.arange(h, device=cos.device).unsqueeze(1).expand(-1, w)
wpos_ids = torch.arange(w, device=cos.device).unsqueeze(0).expand(h, -1)
hpos_ids = (
hpos_ids.reshape(
h // self.spatial_merge_size,
@@ -738,12 +741,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
.flatten()
)
pos_ids = torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)
max_size = max(h, w)
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_size)
pos_ids = pos_ids.to(cos.device, non_blocking=True)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
+3 -3
View File
@@ -609,11 +609,12 @@ class Qwen2VisionTransformer(nn.Module):
def rot_pos_emb(
self, grid_thw: list[list[int]]
) -> tuple[torch.Tensor, torch.Tensor]:
device = self.device
pos_ids = []
max_grid_size = 0
for t, h, w in grid_thw:
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w)
wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1)
hpos_ids = (
hpos_ids.reshape(
h // self.spatial_merge_size,
@@ -641,7 +642,6 @@ class Qwen2VisionTransformer(nn.Module):
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
pos_ids = pos_ids.to(cos.device, non_blocking=True)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
return cos_combined, sin_combined
+3 -2
View File
@@ -94,6 +94,7 @@ from vllm.transformers_utils.processors.qwen3_asr import (
Qwen3ASRProcessor,
)
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
logger = init_logger(__name__)
_ASR_TEXT_TAG = "<asr_text>"
@@ -465,8 +466,8 @@ class Qwen3ASRForConditionalGeneration(
input_features = audio_input["input_features"]
# audio_feature_lengths is keep_on_cpu; the audio tower derives
# device placement from feature_lens, so move it explicitly.
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
input_features.device, non_blocking=True
audio_feature_lengths = async_tensor_h2d(
audio_input["audio_feature_lengths"], input_features.device
)
audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)
@@ -842,9 +842,14 @@ class Qwen3Omni_VisionTransformer(nn.Module):
return self.patch_embed.proj.weight.device
def rot_pos_emb(self, grid_thw):
max_grid_size = grid_thw[:, 1:].max()
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
pos_ids = []
for t, h, w in grid_thw:
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
hpos_ids = torch.arange(h, device=cos.device).unsqueeze(1).expand(-1, w)
hpos_ids = hpos_ids.reshape(
h // self.spatial_merge_size,
self.spatial_merge_size,
@@ -854,7 +859,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
hpos_ids = hpos_ids.permute(0, 2, 1, 3)
hpos_ids = hpos_ids.flatten()
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
wpos_ids = torch.arange(w, device=cos.device).unsqueeze(0).expand(h, -1)
wpos_ids = wpos_ids.reshape(
h // self.spatial_merge_size,
self.spatial_merge_size,
@@ -865,12 +870,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
wpos_ids = wpos_ids.flatten()
pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
pos_ids = torch.cat(pos_ids, dim=0)
max_grid_size = grid_thw[:, 1:].max()
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
pos_ids = pos_ids.to(cos.device, non_blocking=True)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
+2 -1
View File
@@ -56,6 +56,7 @@ from vllm.multimodal.processing import (
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from .clip import dual_encoder_has_text_tokens, merge_dual_encoder_text_and_vision
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
@@ -1010,7 +1011,7 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
flip_indices_cpu = (
starts_cpu[sequence_ids_cpu] + ends_cpu[sequence_ids_cpu]
) - (1 + current_positions_cpu)
flip_indices = flip_indices_cpu.to(features.device, non_blocking=True)
flip_indices = async_tensor_h2d(flip_indices_cpu, features.device)
return features[flip_indices]
+19 -33
View File
@@ -917,16 +917,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
arch,
)
# Preparing persistent buffers
# Since we do not have explicit synchronization in ModelRunnerV2, we do not pin
# reused CPU buffers to avoid a race condition between step N async copies to
# GPU and step N+1 buffer updates.
self.pin_memory = not vllm_config.use_v2_model_runner and PIN_MEMORY
self.paged_kv_indptr = self._make_buffer(max_num_reqs + 1)
self.paged_kv_indptr_cpu_buffer = torch.zeros_like(
self.paged_kv_indptr.cpu, pin_memory=self.pin_memory
) # Extra buffer for mutable paged_kv_indptr.cpu in cuda graph mode
self.paged_kv_indices = self._make_buffer(max_num_pages)
self.paged_kv_last_page_len = self._make_buffer(max_num_reqs)
self.paged_kv_indptr = CpuGpuBuffer(
max_num_reqs + 1, dtype=torch.int32, device=self.device, pin_memory=False
)
self.paged_kv_indices = torch.zeros(
max_num_pages, dtype=torch.int32, device=self.device
)
self.paged_kv_last_page_len = CpuGpuBuffer(
max_num_reqs, dtype=torch.int32, device=self.device, pin_memory=False
)
@property
def kv_cache_layout(self) -> KVCacheLayout:
@@ -970,17 +969,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3")
return self.kv_cache_spec.dtype
def _make_buffer(
self, *size: int | torch.SymInt, dtype: torch.dtype = torch.int32
) -> CpuGpuBuffer:
return CpuGpuBuffer(
*size,
dtype=dtype,
device=self.device,
pin_memory=self.pin_memory,
with_numpy=True,
)
@override # type: ignore[misc]
@classmethod
def get_cudagraph_support(
@@ -1197,7 +1185,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
if decode_wrapper is None:
if use_cudagraph:
paged_kv_indptr = self.paged_kv_indptr.gpu[: batch_size + 1]
paged_kv_indices = self.paged_kv_indices.gpu
paged_kv_indices = self.paged_kv_indices
paged_kv_last_page_len = self.paged_kv_last_page_len.gpu[:batch_size]
else:
paged_kv_indptr = None
@@ -1260,20 +1248,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
dtype=np.int32,
out=self.paged_kv_indptr.np[1 : num_reqs + 1],
)
# NOTE(woosuk): Because self.paged_kv_indptr_cpu can be modified
# after this line (e.g., for cuda graphs), we need to copy the data to
# self.paged_kv_indptr_buffer to avoid race condition.
self.paged_kv_indptr_cpu_buffer[: num_reqs + 1] = self.paged_kv_indptr.cpu[
: num_reqs + 1
]
paged_kv_indptr = self.paged_kv_indptr.gpu[: num_reqs + 1]
paged_kv_indptr.copy_(
self.paged_kv_indptr_cpu_buffer[: num_reqs + 1], non_blocking=True
)
paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_reqs + 1]
if PIN_MEMORY:
paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory()
paged_kv_indptr.copy_(paged_kv_indptr_cpu, non_blocking=True)
# write self.paged_kv_indices inplace
num_actual_pages = self.paged_kv_indptr.np[num_reqs]
paged_kv_indices = self.paged_kv_indices.gpu[:num_actual_pages]
paged_kv_indices = self.paged_kv_indices[:num_actual_pages]
_copy_page_indices_kernel[(num_reqs,)](
paged_kv_indices,
block_table_tensor,
@@ -1289,8 +1272,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
page_size,
paged_kv_last_page_len_np,
)
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[:num_reqs]
if PIN_MEMORY:
paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory()
self.paged_kv_last_page_len.gpu[:num_reqs].copy_(
self.paged_kv_last_page_len.cpu[:num_reqs], non_blocking=True
paged_kv_last_page_len_cpu, non_blocking=True
)
return paged_kv_indices
+12 -4
View File
@@ -11,6 +11,7 @@ import numpy as np
import torch
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
@@ -182,11 +183,18 @@ class LogprobsTensors(NamedTuple):
"""Create empty LogprobsTensors on CPU."""
logprob_token_ids = torch.empty(
(num_positions, num_tokens_per_position), dtype=torch.int32, device="cpu"
(num_positions, num_tokens_per_position),
dtype=torch.int32,
device="cpu",
pin_memory=PIN_MEMORY,
)
logprobs = torch.empty_like(logprob_token_ids, dtype=torch.float32)
selected_token_ranks = torch.empty(
num_positions, dtype=torch.int32, device="cpu"
logprobs = logprob_token_ids.new_empty(
(num_positions, num_tokens_per_position),
dtype=torch.float32,
pin_memory=PIN_MEMORY,
)
selected_token_ranks = logprob_token_ids.new_empty(
num_positions, pin_memory=PIN_MEMORY
)
return LogprobsTensors(
logprob_token_ids=logprob_token_ids,
+4 -1
View File
@@ -215,7 +215,10 @@ class EncoderRunner:
num_computed_tokens = num_computed_tokens + draft_lookahead
is_mm_embed = torch.zeros(
total_num_scheduled_tokens, dtype=torch.bool, device="cpu"
total_num_scheduled_tokens,
dtype=torch.bool,
device="cpu",
pin_memory=PIN_MEMORY,
)
# Whether to gather media embeddings this step.