diff --git a/tests/kernels/attention/test_flashinfer.py b/tests/kernels/attention/test_flashinfer.py index 9a08476976..109f59bc85 100644 --- a/tests/kernels/attention/test_flashinfer.py +++ b/tests/kernels/attention/test_flashinfer.py @@ -190,6 +190,7 @@ def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None: indptr_cpu=kv_indptr, indices=kv_indices, last_page_len_cpu=kv_last_page_lens, + seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"), num_qo_heads=num_query_heads, num_kv_heads=num_kv_heads, head_dim=head_size, @@ -268,6 +269,7 @@ def test_fast_plan_decode_matches_full_plan( indptr_cpu=kv_indptr, indices=kv_indices_buf, last_page_len_cpu=kv_last_page_lens, + seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"), num_qo_heads=num_query_heads, num_kv_heads=num_kv_heads, head_dim=head_size, diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 0e3e38b253..580477ba23 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -49,6 +49,7 @@ from vllm.logger import init_logger from vllm.model_executor.models.interfaces import MixtureOfExperts from vllm.platforms import current_platform from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import PIN_MEMORY from .async_worker import start_async_worker from .eplb_communicator import EplbCommunicator, create_eplb_communicator @@ -1298,6 +1299,8 @@ def _commit_eplb_maps_for_layer( f"Current number of physical experts: {dst.shape[0]}. New number of physical " f"experts {src.shape[0]}." ) + if PIN_MEMORY and src.is_cpu: + src = src.new_empty(src.shape, pin_memory=True).copy_(src) dst.copy_(src, non_blocking=True) num_logical_experts = model_state.logical_to_physical_map.shape[1] @@ -1309,6 +1312,8 @@ def _commit_eplb_maps_for_layer( src = new_replica_count dst = model_state.logical_replica_count[layer] assert src.shape == dst.shape + if PIN_MEMORY: + src = src.pin_memory() dst.copy_(src, non_blocking=True) @@ -1326,12 +1331,14 @@ def _commit_eplb_maps( src = new_physical_to_logical_map dst = model_state.physical_to_logical_map + if PIN_MEMORY and src.is_cpu: + src = src.new_empty(src.shape, pin_memory=True).copy_(src) # Rare Case: When the number of physical experts has changed, discard the old # physical to logical expert map and use the new one. This only happens when the # number of GPUs available to vLLM changes while vLLM is running. Otherwise copy the # new map into the old one. if src.shape[1] != dst.shape[1]: - model_state.physical_to_logical_map = src.to(dst.device) + model_state.physical_to_logical_map = src.to(dst.device, non_blocking=True) else: dst.copy_(src, non_blocking=True) @@ -1346,6 +1353,8 @@ def _commit_eplb_maps( # Commit logical_replica_count src = new_replica_count dst = model_state.logical_replica_count + if PIN_MEMORY: + src = src.pin_memory() dst.copy_(src, non_blocking=True) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py index c94bd2d195..d291d47544 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py @@ -21,6 +21,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( ) from vllm.distributed.parallel_state import get_tensor_model_parallel_rank from vllm.logger import init_logger +from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.core.sched.output import SchedulerOutput @@ -377,10 +378,11 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): with torch.cuda.stream(copy_stream): # Move the CPU slot_mapping to GPU on the copy stream so the # implicit H2D inside fancy indexing doesn't sync the default - # stream. - slot_mapping_gpu = slot_mapping.to( - device=self._kv_cache.device, non_blocking=True - ) + # stream. Deliberate bulk transfer in this reference connector. + with gpu_sync_allowed(): + slot_mapping_gpu = slot_mapping.to( + device=self._kv_cache.device, non_blocking=True + ) hidden_states_gpu = extract_from_kv_cache( self._kv_cache, slot_mapping_gpu, num_tokens ) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 876a625894..00e10e9142 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -265,6 +265,7 @@ from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer from vllm.utils.math_utils import cdiv, round_down, round_up from vllm.utils.torch_utils import ( + PIN_MEMORY, LayerNameType, _encode_layer_name, _resolve_layer_name, @@ -2510,8 +2511,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): query_start_loc_cpu[reqs_start + 1 : num_reqs + 1] - query_start_loc_cpu[reqs_start:num_reqs] ) - context_lens_cpu = ( - seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu + context_lens_cpu = torch.empty( + num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY + ) + torch.subtract( + seq_lens_cpu[reqs_start:num_reqs], + prefill_query_lens_cpu, + out=context_lens_cpu, ) prefill_query_start_loc = ( query_start_loc[reqs_start:] - query_start_loc[reqs_start] diff --git a/vllm/model_executor/layers/fused_moe/modular_kernel.py b/vllm/model_executor/layers/fused_moe/modular_kernel.py index 7d4833a9df..d7446a893a 100644 --- a/vllm/model_executor/layers/fused_moe/modular_kernel.py +++ b/vllm/model_executor/layers/fused_moe/modular_kernel.py @@ -34,6 +34,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, ) from vllm.platforms import current_platform +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.worker.ubatching import ( dbo_enabled, dbo_maybe_run_recv_hook, @@ -109,7 +110,10 @@ class ExpertTokensMetadata: expert_num_tokens_list: list[int], device: str ) -> "ExpertTokensMetadata": expert_num_tokens_cpu = torch.tensor( - expert_num_tokens_list, device="cpu", dtype=torch.int32 + expert_num_tokens_list, + device="cpu", + dtype=torch.int32, + pin_memory=PIN_MEMORY, ) return ExpertTokensMetadata( expert_num_tokens=expert_num_tokens_cpu.to(device, non_blocking=True), diff --git a/vllm/model_executor/models/cosmos3_edge.py b/vllm/model_executor/models/cosmos3_edge.py index 82592751ab..8c5714248b 100644 --- a/vllm/model_executor/models/cosmos3_edge.py +++ b/vllm/model_executor/models/cosmos3_edge.py @@ -21,6 +21,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig +from vllm.utils.torch_utils import async_tensor_h2d from .interfaces import ( MultiModalEmbeddings, @@ -80,10 +81,7 @@ class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer): dim=0, ) lengths_cpu = spatial_shapes.prod(dim=-1).to(torch.int32) - lengths = lengths_cpu.to( - device=pixel_values.device, - non_blocking=True, - ) + lengths = async_tensor_h2d(lengths_cpu, pixel_values.device) cu_seqlens = torch.zeros( lengths.numel() + 1, diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index 06a68f62f3..a421eea976 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -538,7 +538,7 @@ class Ernie4_5_VisionTransformer(nn.Module): else: max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens) - cu_seqlens = cu_seqlens.to(device, non_blocking=True) + cu_seqlens = async_tensor_h2d(cu_seqlens, device) return { "rotary_pos_emb": rotary_pos_emb, @@ -1746,8 +1746,8 @@ class Ernie4_5_VLMoeForConditionalGeneration( # Eager fallback: run the full pipeline (ViT + resampler). The result # is scattered directly, so it must be the post-merge embeddings. pixel_values = mm_kwargs["pixel_values"].type(self.vision_model.dtype) - grid_thw = mm_kwargs["image_grid_thw"].to( - self.vision_model.device, non_blocking=True + grid_thw = async_tensor_h2d( + mm_kwargs["image_grid_thw"], self.vision_model.device ) image_features = self.vision_model(pixel_values, grid_thw) return self.resampler_model(image_features, grid_thw) @@ -1768,7 +1768,7 @@ class Ernie4_5_VLMoeForConditionalGeneration( output = outputs["default"] assert batch_mm_kwargs is not None grid_thw_cpu = batch_mm_kwargs["image_grid_thw"] - grid_thw = grid_thw_cpu.to(output.device, non_blocking=True) + grid_thw = async_tensor_h2d(grid_thw_cpu, output.device) # The valid token count slices the graph output for the eager # resampler call, so it has to come back to the host. num_valid = int( diff --git a/vllm/model_executor/models/glm_ocr.py b/vllm/model_executor/models/glm_ocr.py index 90aca8485d..407b08e8a1 100644 --- a/vllm/model_executor/models/glm_ocr.py +++ b/vllm/model_executor/models/glm_ocr.py @@ -70,6 +70,7 @@ from vllm.model_executor.models.glm4_1v import ( Glm4vVisionTransformer, ) from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.utils.torch_utils import PIN_MEMORY from .utils import ( maybe_prefix, @@ -344,8 +345,11 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer): cu_seqlens = torch.repeat_interleave( grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0] ).cumsum(dim=0, dtype=torch.int32) - cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens]) - cu_seqlens = cu_seqlens.to(self.device, non_blocking=True) + # cat straight into a pinned buffer so the H2D copy stays non-blocking. + cu_seqlens = torch.cat( + [cu_seqlens.new_zeros(1), cu_seqlens], + out=cu_seqlens.new_empty(cu_seqlens.numel() + 1, pin_memory=PIN_MEMORY), + ).to(self.device, non_blocking=True) # pre-compute max_seqlen for attn mask to reduce cuMemcpy operations max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens) diff --git a/vllm/model_executor/models/isaac.py b/vllm/model_executor/models/isaac.py index 1a97288d83..a59345dc25 100644 --- a/vllm/model_executor/models/isaac.py +++ b/vllm/model_executor/models/isaac.py @@ -69,6 +69,7 @@ from vllm.transformers_utils.processors.isaac import ( get_image_size_for_max_num_patches, ) from vllm.utils.tensor_schema import TensorSchema, TensorShape +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from .vision import is_vit_use_data_parallel @@ -970,9 +971,14 @@ class IsaacForConditionalGeneration( device = next(self.language_model.parameters()).device dtype = self.vision_embedding.linear_fc1.weight.dtype pixel_values = pixel_values.to(device=device, dtype=dtype) - spatial_grids = image_grid_thw[:, 1:3].to( - device, dtype=torch.int32, non_blocking=True - ) + # The [:, 1:3] column slice isn't densely laid out, so stage it with a + # single copy straight into pinned memory to keep the H2D non-blocking. + spatial_grids_cpu = torch.empty( + (image_grid_thw.shape[0], 2), + dtype=image_grid_thw.dtype, + pin_memory=PIN_MEMORY, + ).copy_(image_grid_thw[:, 1:3]) + spatial_grids = async_tensor_h2d(spatial_grids_cpu, device, dtype=torch.int32) vision_embeddings = self.vision_embedding((pixel_values, spatial_grids)) merge_size = self.config.vision_config.pixel_shuffle_scale_factor diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index bd7eb79f6f..53cda0ed17 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -59,7 +59,7 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape -from vllm.utils.torch_utils import async_tensor_h2d +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from .interfaces import ( MultiModalEmbeddings, @@ -1305,17 +1305,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): ) else: pixel_values = image_input["pixel_values"].type(self.visual.dtype) - # These are all built on the host, so stage them over - # non-blocking rather than forcing a sync per transfer. - siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to( - pixel_values.device, non_blocking=True - ) + # These are all built on the host; concat straight into pinned + # buffers so the H2D copies stay non-blocking. + siglip_position_ids = torch.concat( + siglip_position_ids, + dim=0, + out=torch.empty( + sum(t.numel() for t in siglip_position_ids), + dtype=torch.int64, + pin_memory=PIN_MEMORY, + ), + ).to(pixel_values.device, non_blocking=True) cu_seqlens = async_tensor_h2d( cu_seqlens, dtype=torch.int32, device=pixel_values.device ) - sample_indices = torch.concat(sample_indices, dim=0).to( - pixel_values.device, non_blocking=True - ) + sample_indices = torch.concat( + sample_indices, + dim=0, + out=torch.empty( + sum(t.numel() for t in sample_indices), + dtype=torch.int64, + pin_memory=PIN_MEMORY, + ), + ).to(pixel_values.device, non_blocking=True) image_embeds = self.visual( pixel_values=pixel_values, @@ -1359,16 +1371,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): ) else: pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - # These are host-built; stage them over non-blocking. - siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to( - pixel_values_videos.device, non_blocking=True - ) + # These are host-built; concat straight into pinned buffers so + # the H2D copies stay non-blocking. + siglip_position_ids = torch.concat( + siglip_position_ids, + dim=0, + out=torch.empty( + sum(t.numel() for t in siglip_position_ids), + dtype=torch.int64, + pin_memory=PIN_MEMORY, + ), + ).to(pixel_values_videos.device, non_blocking=True) cu_seqlens = async_tensor_h2d( cu_seqlens, dtype=torch.int32, device=pixel_values_videos.device ) - sample_indices = torch.concat(sample_indices, dim=0).to( - pixel_values_videos.device, non_blocking=True - ) + sample_indices = torch.concat( + sample_indices, + dim=0, + out=torch.empty( + sum(t.numel() for t in sample_indices), + dtype=torch.int64, + pin_memory=PIN_MEMORY, + ), + ).to(pixel_values_videos.device, non_blocking=True) video_embeds = self.visual( pixel_values=pixel_values_videos, diff --git a/vllm/model_executor/models/paddleocr_vl.py b/vllm/model_executor/models/paddleocr_vl.py index 8c0ca0b62f..4a20531b76 100644 --- a/vllm/model_executor/models/paddleocr_vl.py +++ b/vllm/model_executor/models/paddleocr_vl.py @@ -68,7 +68,7 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape -from vllm.utils.torch_utils import async_tensor_h2d +from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from vllm.v1.attention.backends.registry import AttentionBackendEnum from .ernie45 import Ernie4_5ForCausalLM @@ -1142,10 +1142,17 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support siglip_position_ids.append(image_position_ids) cu_seqlens.append(cu_seqlens[-1] + numel) - # Both are built on the host; stage them over non-blocking. - siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to( - pixel_values.device, non_blocking=True - ) + # Both are built on the host; concat straight into a pinned buffer + # so the H2D copy stays non-blocking. + siglip_position_ids = torch.concat( + siglip_position_ids, + dim=0, + out=torch.empty( + sum(t.numel() for t in siglip_position_ids), + dtype=torch.int64, + pin_memory=PIN_MEMORY, + ), + ).to(pixel_values.device, non_blocking=True) cu_seqlens = async_tensor_h2d( cu_seqlens, dtype=torch.int32, device=pixel_values.device ) diff --git a/vllm/model_executor/models/phi4mm_audio.py b/vllm/model_executor/models/phi4mm_audio.py index ee5d9af679..84abee33c0 100644 --- a/vllm/model_executor/models/phi4mm_audio.py +++ b/vllm/model_executor/models/phi4mm_audio.py @@ -643,8 +643,8 @@ class TransformerEncoderBase(abc.ABC, nn.Module): seq_len, batch_size, chunk_size_nc, left_chunk_nc ) if device.type != "cpu": - enc_streaming_mask_nc = enc_streaming_mask_nc.contiguous().to( - device, non_blocking=True + enc_streaming_mask_nc = async_tensor_h2d( + enc_streaming_mask_nc.contiguous(), device ) if masks is not None: hs_mask_nc = masks & enc_streaming_mask_nc diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 784a5a913e..b09cc504bd 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -91,6 +91,7 @@ from vllm.multimodal.processing.processor import ( from vllm.sequence import IntermediateTensors from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.tensor_schema import TensorSchema, TensorShape +from vllm.utils.torch_utils import async_tensor_h2d from .interfaces import ( MultiModalEmbeddings, @@ -985,8 +986,8 @@ class Qwen2_5OmniConditionalGenerationMixin: input_features = audio_input["input_features"] # audio_feature_lengths is keep_on_cpu; the audio tower derives # device placement from feature_lens, so move it explicitly. - audio_feature_lengths = audio_input["audio_feature_lengths"].to( - input_features.device, non_blocking=True + audio_feature_lengths = async_tensor_h2d( + audio_input["audio_feature_lengths"], input_features.device ) audio_feat_lengths, audio_output_lengths = ( diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index f2bc7aa965..77a463205e 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -1001,7 +1001,7 @@ class Qwen3Omni_VisionTransformer(nn.Module): # Move cu_seqlens to GPU; grid_thw may be on CPU during profile_run # and FA3 vit attention requires cu_seqlens on CUDA. - cu_seqlens = cu_seqlens.to(self.device, non_blocking=True) + cu_seqlens = async_tensor_h2d(cu_seqlens, self.device) hidden_states = hidden_states.unsqueeze(1) rotary_pos_emb_cos = rotary_pos_emb_cos.to(hidden_states.device) rotary_pos_emb_sin = rotary_pos_emb_sin.to(hidden_states.device) @@ -1574,8 +1574,8 @@ class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMix input_features = audio_input["input_features"] # audio_feature_lengths is keep_on_cpu; the audio tower derives # device placement from feature_lens, so move it explicitly. - audio_feature_lengths = audio_input["audio_feature_lengths"].to( - input_features.device, non_blocking=True + audio_feature_lengths = async_tensor_h2d( + audio_input["audio_feature_lengths"], input_features.device ) audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths) diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index dd08f1cc04..74e5402877 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -2367,8 +2367,8 @@ class Qwen3VLForConditionalGeneration( grid_thw_list = grid_thw.tolist() image_embeds_out = [] for emb, size in zip(image_embeds_split, grid_thw_list): - positions = compute_mrope_for_media(size, merge_size).to( - emb.device, non_blocking=True + positions = async_tensor_h2d( + compute_mrope_for_media(size, merge_size), emb.device ) positions = torch.cat( [ @@ -2596,14 +2596,11 @@ class Qwen3VLForConditionalGeneration( identifier="DUMMY", mm_position=PlaceholderRange(offset=0, length=len(unpruned_token_ids)), ) - original_mrope = ( - self.get_mrope_input_positions( - input_tokens=unpruned_token_ids, - mm_features=[mm_feature], - )[0] - .to(device, non_blocking=True) - .permute(1, 0) - ) + original_mrope_cpu = self.get_mrope_input_positions( + input_tokens=unpruned_token_ids, + mm_features=[mm_feature], + )[0] + original_mrope = async_tensor_h2d(original_mrope_cpu, device).permute(1, 0) full_is_video_embed = unpruned_token_ids_tensor == embed_token_id with gpu_sync_allowed(): diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index 248a9f5fce..3b3457c4f0 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -21,6 +21,7 @@ import vllm.envs as envs from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv +from vllm.utils.torch_utils import PIN_MEMORY logger = init_logger(__name__) @@ -181,6 +182,31 @@ def _lazy_import_wrapper( return wrapper +def pin_host_range_buf(length: int) -> None: + """Pin flashinfer's cached host-side arange buffer covering `length`. + + flashinfer's decode plan() builds its host qo_indptr from the cached + `_get_range_buf` entry and copies it to the GPU with non_blocking=True on + every call; the cache entry is unpinned, so the copy silently blocks the + host. Replace the entry with a pinned copy so the copies stay async. + """ + if not PIN_MEMORY: + return + mod = _get_submodule("flashinfer.utils") + if mod is None: + return + get_range_buf = getattr(mod, "_get_range_buf", None) + cache_buf = getattr(mod, "_cache_buf", None) + ceil_pow2 = getattr(mod, "_ceil_pow2", None) + if get_range_buf is None or cache_buf is None or ceil_pow2 is None: + return + get_range_buf(length, "cpu") # Ensure the cache entry exists. + key = (f"range_{ceil_pow2(length)}", "cpu") + buf = cache_buf[key] + if not buf.is_pinned(): + cache_buf[key] = buf.pin_memory() + + # Create lazy wrappers for each function flashinfer_trtllm_bf16_moe = _lazy_import_wrapper( "flashinfer.fused_moe", "trtllm_bf16_moe" diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 07c19ea6b9..c3dfb82a79 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -15,6 +15,7 @@ from flashinfer import ( BatchPrefillWithPagedKVCacheWrapper, BatchPrefillWithRaggedKVCacheWrapper, MultiLevelCascadeAttentionWrapper, + get_seq_lens, ) from flashinfer.decode import fast_decode_plan, trtllm_batch_decode_with_kv_cache from flashinfer.prefill import trtllm_batch_context_with_kv_cache @@ -43,6 +44,7 @@ from vllm.utils.flashinfer import ( can_use_trtllm_attention, flashinfer_xqa_batch_decode_with_kv_cache, force_use_trtllm_attention, + pin_host_range_buf, supports_trtllm_attention, use_trtllm_attention, ) @@ -299,6 +301,7 @@ class BatchDCPPrefillWrapper: paged_kv_indptr_cpu: torch.Tensor, paged_kv_indices: torch.Tensor, paged_kv_last_page_len_cpu: torch.Tensor, + kv_lens_cpu: torch.Tensor, page_size: int, num_qo_heads: int, dcp_world_size: int, @@ -318,6 +321,7 @@ class BatchDCPPrefillWrapper: paged_kv_indptr=paged_kv_indptr_cpu, paged_kv_indices=paged_kv_indices, paged_kv_last_page_len=paged_kv_last_page_len_cpu, + seq_lens=kv_lens_cpu, num_qo_heads=num_qo_heads * dcp_world_size, num_kv_heads=num_kv_heads, head_dim_qk=head_dim, @@ -1580,6 +1584,17 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): paged_kv_last_page_len_prefill_cpu = ( paged_kv_last_page_len_prefill_cpu.pin_memory() ) + # plan() also copies kv_lens to the GPU with non_blocking=True, + # deriving them via get_seq_lens() into unpinned memory when + # seq_lens is not passed; pass an explicit pinned copy of the + # same values so that copy stays async too. + kv_lens_prefill_cpu = get_seq_lens( + paged_kv_indptr_prefill_cpu, + paged_kv_last_page_len_prefill_cpu, + page_size, + ) + if PIN_MEMORY: + kv_lens_prefill_cpu = kv_lens_prefill_cpu.pin_memory() if self.use_dcp: assert isinstance(prefill_wrapper, BatchDCPPrefillWrapper) prefill_wrapper.plan( @@ -1587,6 +1602,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): paged_kv_indptr_cpu=paged_kv_indptr_prefill_cpu, paged_kv_indices=paged_kv_indices, paged_kv_last_page_len_cpu=paged_kv_last_page_len_prefill_cpu, + kv_lens_cpu=kv_lens_prefill_cpu, page_size=self.page_size, num_qo_heads=self.num_qo_heads, dcp_world_size=self.dcp_world_size, @@ -1616,6 +1632,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): paged_kv_indptr=paged_kv_indptr_prefill_cpu, paged_kv_indices=paged_kv_indices, paged_kv_last_page_len=paged_kv_last_page_len_prefill_cpu, + seq_lens=kv_lens_prefill_cpu, num_qo_heads=self.num_qo_heads, num_kv_heads=self.num_kv_heads, head_dim_qk=self.head_dim, @@ -1716,11 +1733,22 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): if PIN_MEMORY: paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory() paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory() + # plan() also copies kv_lens to the GPU with non_blocking=True + # (trtllm-gen/cute-dsl backends), deriving them via + # get_seq_lens() into unpinned memory when seq_lens is not + # passed; pass an explicit pinned copy of the same values + # so that copy stays async too. + kv_lens_decode_cpu = get_seq_lens( + paged_kv_indptr_cpu, paged_kv_last_page_len_cpu, page_size + ) + if PIN_MEMORY: + kv_lens_decode_cpu = kv_lens_decode_cpu.pin_memory() fast_plan_decode( decode_wrapper, indptr_cpu=paged_kv_indptr_cpu, indices=paged_kv_indices, last_page_len_cpu=paged_kv_last_page_len_cpu, + seq_lens_cpu=kv_lens_decode_cpu, num_qo_heads=self.num_qo_heads * self.dcp_world_size, num_kv_heads=self.num_kv_heads, head_dim=self.head_dim, @@ -2579,6 +2607,7 @@ def fast_plan_decode( indptr_cpu: torch.Tensor, indices: torch.Tensor, last_page_len_cpu: torch.Tensor, + seq_lens_cpu: torch.Tensor, num_qo_heads: int, num_kv_heads: int, head_dim: int, @@ -2614,6 +2643,10 @@ def fast_plan_decode( # original plan if we run for dynamic shape. For fixed shape (cudagraph), # this warm up is to generate the _cached_module for the decode wrapper. if not self.is_cuda_graph_enabled or getattr(self, "vllm_first_call", True): + # plan() copies qo_indptr from flashinfer's internal cached host + # arange buffer (_get_range_buf) with non_blocking=True; pin the + # cache entry so the copy stays async. + pin_host_range_buf(len(last_page_len_cpu) + 1) self.plan( indptr=indptr_cpu, indices=indices, @@ -2634,7 +2667,7 @@ def fast_plan_decode( rope_theta=rope_theta, non_blocking=non_blocking, block_tables=None, - seq_lens=None, + seq_lens=seq_lens_cpu, fixed_split_size=fixed_split_size, disable_split_kv=disable_split_kv, ) diff --git a/vllm/v1/attention/backends/hpc_attn.py b/vllm/v1/attention/backends/hpc_attn.py index f83d599455..14545307fe 100644 --- a/vllm/v1/attention/backends/hpc_attn.py +++ b/vllm/v1/attention/backends/hpc_attn.py @@ -19,6 +19,7 @@ from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -216,8 +217,13 @@ class HpcAttnMetadataBuilder(AttentionMetadataBuilder[HpcAttnMetadata]): if num_prefills > 0: qo_indptr_cpu = common_attn_metadata.query_start_loc_cpu prefill_start = num_decodes - qo_indptr_prefill_cpu = ( - qo_indptr_cpu[prefill_start:] - qo_indptr_cpu[prefill_start] + qo_indptr_prefill_cpu = torch.empty( + num_prefills + 1, dtype=torch.int32, pin_memory=PIN_MEMORY + ) + torch.subtract( + qo_indptr_cpu[prefill_start:], + qo_indptr_cpu[prefill_start], + out=qo_indptr_prefill_cpu, ) qo_indptr = qo_indptr_prefill_cpu.to(self.device, non_blocking=True) diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index a73fb6c4b4..cbae01519b 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -21,7 +21,11 @@ from vllm.platforms.interface import DeviceCapability from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.math_utils import cdiv from vllm.utils.platform_utils import num_compute_units -from vllm.utils.torch_utils import get_dtype_size, is_quantized_kv_cache +from vllm.utils.torch_utils import ( + async_tensor_h2d, + get_dtype_size, + is_quantized_kv_cache, +) from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -593,6 +597,7 @@ class AiterFlashAttentionMetadataBuilder( num_extends + 1, dtype=torch.int32, device=seq_lens_for_extend.device, + pin_memory=True, ) torch.cumsum( swa_seqlen_for_extend, @@ -623,8 +628,8 @@ class AiterFlashAttentionMetadataBuilder( swa_metadata = AiterChunkSlidingWindowMetadata( swa_cu_seqlens=cu_seq_lens.to(self.device, non_blocking=True), - swa_seq_starts=seq_starts.to(self.device, non_blocking=True), - swa_token_to_batch=token_to_seq.to(self.device, non_blocking=True), + swa_seq_starts=async_tensor_h2d(seq_starts, self.device), + swa_token_to_batch=async_tensor_h2d(token_to_seq, self.device), swa_max_seqlens=max_seqlen_k, swa_total_tokens=total_tokens, swa_workspace=swa_workspace, @@ -668,8 +673,8 @@ class AiterFlashAttentionMetadataBuilder( chunk_context_metadata = AiterChunkContextMetadata( workspace=self.extend_workspace, cu_seq_lens_chunk=cu_seq_lens_cpu.to(self.device, non_blocking=True), - chunk_starts=chunk_starts.to(self.device, non_blocking=True), - token_to_batch=token_to_batch_tensor.to(self.device, non_blocking=True), + chunk_starts=async_tensor_h2d(chunk_starts, self.device), + token_to_batch=async_tensor_h2d(token_to_batch_tensor, self.device), max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(), num_chunks=num_chunks, total_token_per_batch=cu_seq_lens_cpu[:, -1].tolist(), diff --git a/vllm/v1/worker/dp_utils.py b/vllm/v1/worker/dp_utils.py index a7476bffa4..07bda70483 100644 --- a/vllm/v1/worker/dp_utils.py +++ b/vllm/v1/worker/dp_utils.py @@ -10,6 +10,7 @@ from vllm.config import ParallelConfig from vllm.distributed.parallel_state import get_dp_group from vllm.logger import init_logger from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.worker.ubatch_utils import ( check_ubatch_thresholds, is_last_ubatch_empty, @@ -47,7 +48,8 @@ def _run_ar( dp_rank = parallel_config.data_parallel_rank device, group = _get_device_and_group(parallel_config) # Populate this rank's contribution on CPU to reduce GPU syncs. - tensor_cpu = torch.zeros(4, dp_size, dtype=torch.int32) + pin_memory = PIN_MEMORY and torch.device(device).type == "cuda" + tensor_cpu = torch.zeros(4, dp_size, dtype=torch.int32, pin_memory=pin_memory) tensor_cpu[0][dp_rank] = orig_num_tokens_per_ubatch tensor_cpu[1][dp_rank] = padded_num_tokens_per_ubatch tensor_cpu[2][dp_rank] = 1 if should_ubatch else 0 diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 72981125ac..6d35dc73ca 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2258,8 +2258,10 @@ class GPUModelRunner( if self.use_async_spec_decode and (self.uses_mrope or self.uses_xdrope_dim > 0): drift = self.num_computed_tokens[req_indices_gpu].to( torch.int64 - ) - self.input_batch.num_computed_tokens_cpu_tensor[req_indices].to( - device=self.device, dtype=torch.int64, non_blocking=True + ) - async_tensor_h2d( + self.input_batch.num_computed_tokens_cpu_tensor[req_indices], + device=self.device, + dtype=torch.int64, ) target = self.mrope_positions if self.uses_mrope else self.xdrope_positions target.gpu[:, :total_num_scheduled_tokens] += drift @@ -6324,9 +6326,7 @@ class GPUModelRunner( self.eplb_step(is_dummy=True, is_profile=is_profile) logit_indices = np.cumsum(num_scheduled_tokens) - 1 - logit_indices_device = torch.from_numpy(logit_indices).to( - self.device, non_blocking=True - ) + logit_indices_device = async_tensor_h2d(logit_indices, device=self.device) return hidden_states, hidden_states[logit_indices_device] @torch.inference_mode()