[Perf] Ensure async h2d copies are pinned in more places (#55202)

This commit is contained in:
Nick Hill
2026-09-04 17:17:01 -07:00
committed by GitHub
parent 874df9373d
commit 8277c42e4c
21 changed files with 206 additions and 73 deletions
@@ -190,6 +190,7 @@ def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
indptr_cpu=kv_indptr,
indices=kv_indices,
last_page_len_cpu=kv_last_page_lens,
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
num_qo_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
@@ -268,6 +269,7 @@ def test_fast_plan_decode_matches_full_plan(
indptr_cpu=kv_indptr,
indices=kv_indices_buf,
last_page_len_cpu=kv_last_page_lens,
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
num_qo_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
+10 -1
View File
@@ -49,6 +49,7 @@ from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import PIN_MEMORY
from .async_worker import start_async_worker
from .eplb_communicator import EplbCommunicator, create_eplb_communicator
@@ -1298,6 +1299,8 @@ def _commit_eplb_maps_for_layer(
f"Current number of physical experts: {dst.shape[0]}. New number of physical "
f"experts {src.shape[0]}."
)
if PIN_MEMORY and src.is_cpu:
src = src.new_empty(src.shape, pin_memory=True).copy_(src)
dst.copy_(src, non_blocking=True)
num_logical_experts = model_state.logical_to_physical_map.shape[1]
@@ -1309,6 +1312,8 @@ def _commit_eplb_maps_for_layer(
src = new_replica_count
dst = model_state.logical_replica_count[layer]
assert src.shape == dst.shape
if PIN_MEMORY:
src = src.pin_memory()
dst.copy_(src, non_blocking=True)
@@ -1326,12 +1331,14 @@ def _commit_eplb_maps(
src = new_physical_to_logical_map
dst = model_state.physical_to_logical_map
if PIN_MEMORY and src.is_cpu:
src = src.new_empty(src.shape, pin_memory=True).copy_(src)
# Rare Case: When the number of physical experts has changed, discard the old
# physical to logical expert map and use the new one. This only happens when the
# number of GPUs available to vLLM changes while vLLM is running. Otherwise copy the
# new map into the old one.
if src.shape[1] != dst.shape[1]:
model_state.physical_to_logical_map = src.to(dst.device)
model_state.physical_to_logical_map = src.to(dst.device, non_blocking=True)
else:
dst.copy_(src, non_blocking=True)
@@ -1346,6 +1353,8 @@ def _commit_eplb_maps(
# Commit logical_replica_count
src = new_replica_count
dst = model_state.logical_replica_count
if PIN_MEMORY:
src = src.pin_memory()
dst.copy_(src, non_blocking=True)
@@ -21,6 +21,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
)
from vllm.distributed.parallel_state import get_tensor_model_parallel_rank
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
@@ -377,10 +378,11 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
with torch.cuda.stream(copy_stream):
# Move the CPU slot_mapping to GPU on the copy stream so the
# implicit H2D inside fancy indexing doesn't sync the default
# stream.
slot_mapping_gpu = slot_mapping.to(
device=self._kv_cache.device, non_blocking=True
)
# stream. Deliberate bulk transfer in this reference connector.
with gpu_sync_allowed():
slot_mapping_gpu = slot_mapping.to(
device=self._kv_cache.device, non_blocking=True
)
hidden_states_gpu = extract_from_kv_cache(
self._kv_cache, slot_mapping_gpu, num_tokens
)
@@ -265,6 +265,7 @@ from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.math_utils import cdiv, round_down, round_up
from vllm.utils.torch_utils import (
PIN_MEMORY,
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
@@ -2510,8 +2511,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
query_start_loc_cpu[reqs_start + 1 : num_reqs + 1]
- query_start_loc_cpu[reqs_start:num_reqs]
)
context_lens_cpu = (
seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu
context_lens_cpu = torch.empty(
num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY
)
torch.subtract(
seq_lens_cpu[reqs_start:num_reqs],
prefill_query_lens_cpu,
out=context_lens_cpu,
)
prefill_query_start_loc = (
query_start_loc[reqs_start:] - query_start_loc[reqs_start]
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.worker.ubatching import (
dbo_enabled,
dbo_maybe_run_recv_hook,
@@ -109,7 +110,10 @@ class ExpertTokensMetadata:
expert_num_tokens_list: list[int], device: str
) -> "ExpertTokensMetadata":
expert_num_tokens_cpu = torch.tensor(
expert_num_tokens_list, device="cpu", dtype=torch.int32
expert_num_tokens_list,
device="cpu",
dtype=torch.int32,
pin_memory=PIN_MEMORY,
)
return ExpertTokensMetadata(
expert_num_tokens=expert_num_tokens_cpu.to(device, non_blocking=True),
+2 -4
View File
@@ -21,6 +21,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -80,10 +81,7 @@ class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer):
dim=0,
)
lengths_cpu = spatial_shapes.prod(dim=-1).to(torch.int32)
lengths = lengths_cpu.to(
device=pixel_values.device,
non_blocking=True,
)
lengths = async_tensor_h2d(lengths_cpu, pixel_values.device)
cu_seqlens = torch.zeros(
lengths.numel() + 1,
+4 -4
View File
@@ -538,7 +538,7 @@ class Ernie4_5_VisionTransformer(nn.Module):
else:
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
cu_seqlens = cu_seqlens.to(device, non_blocking=True)
cu_seqlens = async_tensor_h2d(cu_seqlens, device)
return {
"rotary_pos_emb": rotary_pos_emb,
@@ -1746,8 +1746,8 @@ class Ernie4_5_VLMoeForConditionalGeneration(
# Eager fallback: run the full pipeline (ViT + resampler). The result
# is scattered directly, so it must be the post-merge embeddings.
pixel_values = mm_kwargs["pixel_values"].type(self.vision_model.dtype)
grid_thw = mm_kwargs["image_grid_thw"].to(
self.vision_model.device, non_blocking=True
grid_thw = async_tensor_h2d(
mm_kwargs["image_grid_thw"], self.vision_model.device
)
image_features = self.vision_model(pixel_values, grid_thw)
return self.resampler_model(image_features, grid_thw)
@@ -1768,7 +1768,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
output = outputs["default"]
assert batch_mm_kwargs is not None
grid_thw_cpu = batch_mm_kwargs["image_grid_thw"]
grid_thw = grid_thw_cpu.to(output.device, non_blocking=True)
grid_thw = async_tensor_h2d(grid_thw_cpu, output.device)
# The valid token count slices the graph output for the eager
# resampler call, so it has to come back to the host.
num_valid = int(
+6 -2
View File
@@ -70,6 +70,7 @@ from vllm.model_executor.models.glm4_1v import (
Glm4vVisionTransformer,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.utils.torch_utils import PIN_MEMORY
from .utils import (
maybe_prefix,
@@ -344,8 +345,11 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
).cumsum(dim=0, dtype=torch.int32)
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
# cat straight into a pinned buffer so the H2D copy stays non-blocking.
cu_seqlens = torch.cat(
[cu_seqlens.new_zeros(1), cu_seqlens],
out=cu_seqlens.new_empty(cu_seqlens.numel() + 1, pin_memory=PIN_MEMORY),
).to(self.device, non_blocking=True)
# pre-compute max_seqlen for attn mask to reduce cuMemcpy operations
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
+9 -3
View File
@@ -69,6 +69,7 @@ from vllm.transformers_utils.processors.isaac import (
get_image_size_for_max_num_patches,
)
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from .vision import is_vit_use_data_parallel
@@ -970,9 +971,14 @@ class IsaacForConditionalGeneration(
device = next(self.language_model.parameters()).device
dtype = self.vision_embedding.linear_fc1.weight.dtype
pixel_values = pixel_values.to(device=device, dtype=dtype)
spatial_grids = image_grid_thw[:, 1:3].to(
device, dtype=torch.int32, non_blocking=True
)
# The [:, 1:3] column slice isn't densely laid out, so stage it with a
# single copy straight into pinned memory to keep the H2D non-blocking.
spatial_grids_cpu = torch.empty(
(image_grid_thw.shape[0], 2),
dtype=image_grid_thw.dtype,
pin_memory=PIN_MEMORY,
).copy_(image_grid_thw[:, 1:3])
spatial_grids = async_tensor_h2d(spatial_grids_cpu, device, dtype=torch.int32)
vision_embeddings = self.vision_embedding((pixel_values, spatial_grids))
merge_size = self.config.vision_config.pixel_shuffle_scale_factor
+41 -16
View File
@@ -59,7 +59,7 @@ from vllm.multimodal.processing import (
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -1305,17 +1305,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
)
else:
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
# These are all built on the host, so stage them over
# non-blocking rather than forcing a sync per transfer.
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
pixel_values.device, non_blocking=True
)
# These are all built on the host; concat straight into pinned
# buffers so the H2D copies stay non-blocking.
siglip_position_ids = torch.concat(
siglip_position_ids,
dim=0,
out=torch.empty(
sum(t.numel() for t in siglip_position_ids),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(
cu_seqlens, dtype=torch.int32, device=pixel_values.device
)
sample_indices = torch.concat(sample_indices, dim=0).to(
pixel_values.device, non_blocking=True
)
sample_indices = torch.concat(
sample_indices,
dim=0,
out=torch.empty(
sum(t.numel() for t in sample_indices),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values.device, non_blocking=True)
image_embeds = self.visual(
pixel_values=pixel_values,
@@ -1359,16 +1371,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
)
else:
pixel_values_videos = pixel_values_videos.type(self.visual.dtype)
# These are host-built; stage them over non-blocking.
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
pixel_values_videos.device, non_blocking=True
)
# These are host-built; concat straight into pinned buffers so
# the H2D copies stay non-blocking.
siglip_position_ids = torch.concat(
siglip_position_ids,
dim=0,
out=torch.empty(
sum(t.numel() for t in siglip_position_ids),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values_videos.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(
cu_seqlens, dtype=torch.int32, device=pixel_values_videos.device
)
sample_indices = torch.concat(sample_indices, dim=0).to(
pixel_values_videos.device, non_blocking=True
)
sample_indices = torch.concat(
sample_indices,
dim=0,
out=torch.empty(
sum(t.numel() for t in sample_indices),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values_videos.device, non_blocking=True)
video_embeds = self.visual(
pixel_values=pixel_values_videos,
+12 -5
View File
@@ -68,7 +68,7 @@ from vllm.multimodal.processing import (
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .ernie45 import Ernie4_5ForCausalLM
@@ -1142,10 +1142,17 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
siglip_position_ids.append(image_position_ids)
cu_seqlens.append(cu_seqlens[-1] + numel)
# Both are built on the host; stage them over non-blocking.
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
pixel_values.device, non_blocking=True
)
# Both are built on the host; concat straight into a pinned buffer
# so the H2D copy stays non-blocking.
siglip_position_ids = torch.concat(
siglip_position_ids,
dim=0,
out=torch.empty(
sum(t.numel() for t in siglip_position_ids),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(
cu_seqlens, dtype=torch.int32, device=pixel_values.device
)
+2 -2
View File
@@ -643,8 +643,8 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
seq_len, batch_size, chunk_size_nc, left_chunk_nc
)
if device.type != "cpu":
enc_streaming_mask_nc = enc_streaming_mask_nc.contiguous().to(
device, non_blocking=True
enc_streaming_mask_nc = async_tensor_h2d(
enc_streaming_mask_nc.contiguous(), device
)
if masks is not None:
hs_mask_nc = masks & enc_streaming_mask_nc
@@ -91,6 +91,7 @@ from vllm.multimodal.processing.processor import (
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -985,8 +986,8 @@ class Qwen2_5OmniConditionalGenerationMixin:
input_features = audio_input["input_features"]
# audio_feature_lengths is keep_on_cpu; the audio tower derives
# device placement from feature_lens, so move it explicitly.
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
input_features.device, non_blocking=True
audio_feature_lengths = async_tensor_h2d(
audio_input["audio_feature_lengths"], input_features.device
)
audio_feat_lengths, audio_output_lengths = (
@@ -1001,7 +1001,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
# Move cu_seqlens to GPU; grid_thw may be on CPU during profile_run
# and FA3 vit attention requires cu_seqlens on CUDA.
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(cu_seqlens, self.device)
hidden_states = hidden_states.unsqueeze(1)
rotary_pos_emb_cos = rotary_pos_emb_cos.to(hidden_states.device)
rotary_pos_emb_sin = rotary_pos_emb_sin.to(hidden_states.device)
@@ -1574,8 +1574,8 @@ class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMix
input_features = audio_input["input_features"]
# audio_feature_lengths is keep_on_cpu; the audio tower derives
# device placement from feature_lens, so move it explicitly.
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
input_features.device, non_blocking=True
audio_feature_lengths = async_tensor_h2d(
audio_input["audio_feature_lengths"], input_features.device
)
audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)
+7 -10
View File
@@ -2367,8 +2367,8 @@ class Qwen3VLForConditionalGeneration(
grid_thw_list = grid_thw.tolist()
image_embeds_out = []
for emb, size in zip(image_embeds_split, grid_thw_list):
positions = compute_mrope_for_media(size, merge_size).to(
emb.device, non_blocking=True
positions = async_tensor_h2d(
compute_mrope_for_media(size, merge_size), emb.device
)
positions = torch.cat(
[
@@ -2596,14 +2596,11 @@ class Qwen3VLForConditionalGeneration(
identifier="DUMMY",
mm_position=PlaceholderRange(offset=0, length=len(unpruned_token_ids)),
)
original_mrope = (
self.get_mrope_input_positions(
input_tokens=unpruned_token_ids,
mm_features=[mm_feature],
)[0]
.to(device, non_blocking=True)
.permute(1, 0)
)
original_mrope_cpu = self.get_mrope_input_positions(
input_tokens=unpruned_token_ids,
mm_features=[mm_feature],
)[0]
original_mrope = async_tensor_h2d(original_mrope_cpu, device).permute(1, 0)
full_is_video_embed = unpruned_token_ids_tensor == embed_token_id
with gpu_sync_allowed():
+26
View File
@@ -21,6 +21,7 @@ import vllm.envs as envs
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import PIN_MEMORY
logger = init_logger(__name__)
@@ -181,6 +182,31 @@ def _lazy_import_wrapper(
return wrapper
def pin_host_range_buf(length: int) -> None:
"""Pin flashinfer's cached host-side arange buffer covering `length`.
flashinfer's decode plan() builds its host qo_indptr from the cached
`_get_range_buf` entry and copies it to the GPU with non_blocking=True on
every call; the cache entry is unpinned, so the copy silently blocks the
host. Replace the entry with a pinned copy so the copies stay async.
"""
if not PIN_MEMORY:
return
mod = _get_submodule("flashinfer.utils")
if mod is None:
return
get_range_buf = getattr(mod, "_get_range_buf", None)
cache_buf = getattr(mod, "_cache_buf", None)
ceil_pow2 = getattr(mod, "_ceil_pow2", None)
if get_range_buf is None or cache_buf is None or ceil_pow2 is None:
return
get_range_buf(length, "cpu") # Ensure the cache entry exists.
key = (f"range_{ceil_pow2(length)}", "cpu")
buf = cache_buf[key]
if not buf.is_pinned():
cache_buf[key] = buf.pin_memory()
# Create lazy wrappers for each function
flashinfer_trtllm_bf16_moe = _lazy_import_wrapper(
"flashinfer.fused_moe", "trtllm_bf16_moe"
+34 -1
View File
@@ -15,6 +15,7 @@ from flashinfer import (
BatchPrefillWithPagedKVCacheWrapper,
BatchPrefillWithRaggedKVCacheWrapper,
MultiLevelCascadeAttentionWrapper,
get_seq_lens,
)
from flashinfer.decode import fast_decode_plan, trtllm_batch_decode_with_kv_cache
from flashinfer.prefill import trtllm_batch_context_with_kv_cache
@@ -43,6 +44,7 @@ from vllm.utils.flashinfer import (
can_use_trtllm_attention,
flashinfer_xqa_batch_decode_with_kv_cache,
force_use_trtllm_attention,
pin_host_range_buf,
supports_trtllm_attention,
use_trtllm_attention,
)
@@ -299,6 +301,7 @@ class BatchDCPPrefillWrapper:
paged_kv_indptr_cpu: torch.Tensor,
paged_kv_indices: torch.Tensor,
paged_kv_last_page_len_cpu: torch.Tensor,
kv_lens_cpu: torch.Tensor,
page_size: int,
num_qo_heads: int,
dcp_world_size: int,
@@ -318,6 +321,7 @@ class BatchDCPPrefillWrapper:
paged_kv_indptr=paged_kv_indptr_cpu,
paged_kv_indices=paged_kv_indices,
paged_kv_last_page_len=paged_kv_last_page_len_cpu,
seq_lens=kv_lens_cpu,
num_qo_heads=num_qo_heads * dcp_world_size,
num_kv_heads=num_kv_heads,
head_dim_qk=head_dim,
@@ -1580,6 +1584,17 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
paged_kv_last_page_len_prefill_cpu = (
paged_kv_last_page_len_prefill_cpu.pin_memory()
)
# plan() also copies kv_lens to the GPU with non_blocking=True,
# deriving them via get_seq_lens() into unpinned memory when
# seq_lens is not passed; pass an explicit pinned copy of the
# same values so that copy stays async too.
kv_lens_prefill_cpu = get_seq_lens(
paged_kv_indptr_prefill_cpu,
paged_kv_last_page_len_prefill_cpu,
page_size,
)
if PIN_MEMORY:
kv_lens_prefill_cpu = kv_lens_prefill_cpu.pin_memory()
if self.use_dcp:
assert isinstance(prefill_wrapper, BatchDCPPrefillWrapper)
prefill_wrapper.plan(
@@ -1587,6 +1602,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
paged_kv_indptr_cpu=paged_kv_indptr_prefill_cpu,
paged_kv_indices=paged_kv_indices,
paged_kv_last_page_len_cpu=paged_kv_last_page_len_prefill_cpu,
kv_lens_cpu=kv_lens_prefill_cpu,
page_size=self.page_size,
num_qo_heads=self.num_qo_heads,
dcp_world_size=self.dcp_world_size,
@@ -1616,6 +1632,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
paged_kv_indptr=paged_kv_indptr_prefill_cpu,
paged_kv_indices=paged_kv_indices,
paged_kv_last_page_len=paged_kv_last_page_len_prefill_cpu,
seq_lens=kv_lens_prefill_cpu,
num_qo_heads=self.num_qo_heads,
num_kv_heads=self.num_kv_heads,
head_dim_qk=self.head_dim,
@@ -1716,11 +1733,22 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
if PIN_MEMORY:
paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory()
paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory()
# plan() also copies kv_lens to the GPU with non_blocking=True
# (trtllm-gen/cute-dsl backends), deriving them via
# get_seq_lens() into unpinned memory when seq_lens is not
# passed; pass an explicit pinned copy of the same values
# so that copy stays async too.
kv_lens_decode_cpu = get_seq_lens(
paged_kv_indptr_cpu, paged_kv_last_page_len_cpu, page_size
)
if PIN_MEMORY:
kv_lens_decode_cpu = kv_lens_decode_cpu.pin_memory()
fast_plan_decode(
decode_wrapper,
indptr_cpu=paged_kv_indptr_cpu,
indices=paged_kv_indices,
last_page_len_cpu=paged_kv_last_page_len_cpu,
seq_lens_cpu=kv_lens_decode_cpu,
num_qo_heads=self.num_qo_heads * self.dcp_world_size,
num_kv_heads=self.num_kv_heads,
head_dim=self.head_dim,
@@ -2579,6 +2607,7 @@ def fast_plan_decode(
indptr_cpu: torch.Tensor,
indices: torch.Tensor,
last_page_len_cpu: torch.Tensor,
seq_lens_cpu: torch.Tensor,
num_qo_heads: int,
num_kv_heads: int,
head_dim: int,
@@ -2614,6 +2643,10 @@ def fast_plan_decode(
# original plan if we run for dynamic shape. For fixed shape (cudagraph),
# this warm up is to generate the _cached_module for the decode wrapper.
if not self.is_cuda_graph_enabled or getattr(self, "vllm_first_call", True):
# plan() copies qo_indptr from flashinfer's internal cached host
# arange buffer (_get_range_buf) with non_blocking=True; pin the
# cache entry so the copy stays async.
pin_host_range_buf(len(last_page_len_cpu) + 1)
self.plan(
indptr=indptr_cpu,
indices=indices,
@@ -2634,7 +2667,7 @@ def fast_plan_decode(
rope_theta=rope_theta,
non_blocking=non_blocking,
block_tables=None,
seq_lens=None,
seq_lens=seq_lens_cpu,
fixed_split_size=fixed_split_size,
disable_split_kv=disable_split_kv,
)
+8 -2
View File
@@ -19,6 +19,7 @@ from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -216,8 +217,13 @@ class HpcAttnMetadataBuilder(AttentionMetadataBuilder[HpcAttnMetadata]):
if num_prefills > 0:
qo_indptr_cpu = common_attn_metadata.query_start_loc_cpu
prefill_start = num_decodes
qo_indptr_prefill_cpu = (
qo_indptr_cpu[prefill_start:] - qo_indptr_cpu[prefill_start]
qo_indptr_prefill_cpu = torch.empty(
num_prefills + 1, dtype=torch.int32, pin_memory=PIN_MEMORY
)
torch.subtract(
qo_indptr_cpu[prefill_start:],
qo_indptr_cpu[prefill_start],
out=qo_indptr_prefill_cpu,
)
qo_indptr = qo_indptr_prefill_cpu.to(self.device, non_blocking=True)
+10 -5
View File
@@ -21,7 +21,11 @@ from vllm.platforms.interface import DeviceCapability
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import get_dtype_size, is_quantized_kv_cache
from vllm.utils.torch_utils import (
async_tensor_h2d,
get_dtype_size,
is_quantized_kv_cache,
)
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -593,6 +597,7 @@ class AiterFlashAttentionMetadataBuilder(
num_extends + 1,
dtype=torch.int32,
device=seq_lens_for_extend.device,
pin_memory=True,
)
torch.cumsum(
swa_seqlen_for_extend,
@@ -623,8 +628,8 @@ class AiterFlashAttentionMetadataBuilder(
swa_metadata = AiterChunkSlidingWindowMetadata(
swa_cu_seqlens=cu_seq_lens.to(self.device, non_blocking=True),
swa_seq_starts=seq_starts.to(self.device, non_blocking=True),
swa_token_to_batch=token_to_seq.to(self.device, non_blocking=True),
swa_seq_starts=async_tensor_h2d(seq_starts, self.device),
swa_token_to_batch=async_tensor_h2d(token_to_seq, self.device),
swa_max_seqlens=max_seqlen_k,
swa_total_tokens=total_tokens,
swa_workspace=swa_workspace,
@@ -668,8 +673,8 @@ class AiterFlashAttentionMetadataBuilder(
chunk_context_metadata = AiterChunkContextMetadata(
workspace=self.extend_workspace,
cu_seq_lens_chunk=cu_seq_lens_cpu.to(self.device, non_blocking=True),
chunk_starts=chunk_starts.to(self.device, non_blocking=True),
token_to_batch=token_to_batch_tensor.to(self.device, non_blocking=True),
chunk_starts=async_tensor_h2d(chunk_starts, self.device),
token_to_batch=async_tensor_h2d(token_to_batch_tensor, self.device),
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
num_chunks=num_chunks,
total_token_per_batch=cu_seq_lens_cpu[:, -1].tolist(),
+3 -1
View File
@@ -10,6 +10,7 @@ from vllm.config import ParallelConfig
from vllm.distributed.parallel_state import get_dp_group
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.worker.ubatch_utils import (
check_ubatch_thresholds,
is_last_ubatch_empty,
@@ -47,7 +48,8 @@ def _run_ar(
dp_rank = parallel_config.data_parallel_rank
device, group = _get_device_and_group(parallel_config)
# Populate this rank's contribution on CPU to reduce GPU syncs.
tensor_cpu = torch.zeros(4, dp_size, dtype=torch.int32)
pin_memory = PIN_MEMORY and torch.device(device).type == "cuda"
tensor_cpu = torch.zeros(4, dp_size, dtype=torch.int32, pin_memory=pin_memory)
tensor_cpu[0][dp_rank] = orig_num_tokens_per_ubatch
tensor_cpu[1][dp_rank] = padded_num_tokens_per_ubatch
tensor_cpu[2][dp_rank] = 1 if should_ubatch else 0
+5 -5
View File
@@ -2258,8 +2258,10 @@ class GPUModelRunner(
if self.use_async_spec_decode and (self.uses_mrope or self.uses_xdrope_dim > 0):
drift = self.num_computed_tokens[req_indices_gpu].to(
torch.int64
) - self.input_batch.num_computed_tokens_cpu_tensor[req_indices].to(
device=self.device, dtype=torch.int64, non_blocking=True
) - async_tensor_h2d(
self.input_batch.num_computed_tokens_cpu_tensor[req_indices],
device=self.device,
dtype=torch.int64,
)
target = self.mrope_positions if self.uses_mrope else self.xdrope_positions
target.gpu[:, :total_num_scheduled_tokens] += drift
@@ -6324,9 +6326,7 @@ class GPUModelRunner(
self.eplb_step(is_dummy=True, is_profile=is_profile)
logit_indices = np.cumsum(num_scheduled_tokens) - 1
logit_indices_device = torch.from_numpy(logit_indices).to(
self.device, non_blocking=True
)
logit_indices_device = async_tensor_h2d(logit_indices, device=self.device)
return hidden_states, hidden_states[logit_indices_device]
@torch.inference_mode()