[Perf] Ensure async h2d copies are pinned in more places (#55202)
This commit is contained in:
@@ -190,6 +190,7 @@ def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
|
||||
indptr_cpu=kv_indptr,
|
||||
indices=kv_indices,
|
||||
last_page_len_cpu=kv_last_page_lens,
|
||||
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
|
||||
num_qo_heads=num_query_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_size,
|
||||
@@ -268,6 +269,7 @@ def test_fast_plan_decode_matches_full_plan(
|
||||
indptr_cpu=kv_indptr,
|
||||
indices=kv_indices_buf,
|
||||
last_page_len_cpu=kv_last_page_lens,
|
||||
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
|
||||
num_qo_heads=num_query_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_size,
|
||||
|
||||
@@ -49,6 +49,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
from .async_worker import start_async_worker
|
||||
from .eplb_communicator import EplbCommunicator, create_eplb_communicator
|
||||
@@ -1298,6 +1299,8 @@ def _commit_eplb_maps_for_layer(
|
||||
f"Current number of physical experts: {dst.shape[0]}. New number of physical "
|
||||
f"experts {src.shape[0]}."
|
||||
)
|
||||
if PIN_MEMORY and src.is_cpu:
|
||||
src = src.new_empty(src.shape, pin_memory=True).copy_(src)
|
||||
dst.copy_(src, non_blocking=True)
|
||||
|
||||
num_logical_experts = model_state.logical_to_physical_map.shape[1]
|
||||
@@ -1309,6 +1312,8 @@ def _commit_eplb_maps_for_layer(
|
||||
src = new_replica_count
|
||||
dst = model_state.logical_replica_count[layer]
|
||||
assert src.shape == dst.shape
|
||||
if PIN_MEMORY:
|
||||
src = src.pin_memory()
|
||||
dst.copy_(src, non_blocking=True)
|
||||
|
||||
|
||||
@@ -1326,12 +1331,14 @@ def _commit_eplb_maps(
|
||||
src = new_physical_to_logical_map
|
||||
dst = model_state.physical_to_logical_map
|
||||
|
||||
if PIN_MEMORY and src.is_cpu:
|
||||
src = src.new_empty(src.shape, pin_memory=True).copy_(src)
|
||||
# Rare Case: When the number of physical experts has changed, discard the old
|
||||
# physical to logical expert map and use the new one. This only happens when the
|
||||
# number of GPUs available to vLLM changes while vLLM is running. Otherwise copy the
|
||||
# new map into the old one.
|
||||
if src.shape[1] != dst.shape[1]:
|
||||
model_state.physical_to_logical_map = src.to(dst.device)
|
||||
model_state.physical_to_logical_map = src.to(dst.device, non_blocking=True)
|
||||
else:
|
||||
dst.copy_(src, non_blocking=True)
|
||||
|
||||
@@ -1346,6 +1353,8 @@ def _commit_eplb_maps(
|
||||
# Commit logical_replica_count
|
||||
src = new_replica_count
|
||||
dst = model_state.logical_replica_count
|
||||
if PIN_MEMORY:
|
||||
src = src.pin_memory()
|
||||
dst.copy_(src, non_blocking=True)
|
||||
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
)
|
||||
from vllm.distributed.parallel_state import get_tensor_model_parallel_rank
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
@@ -377,10 +378,11 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
with torch.cuda.stream(copy_stream):
|
||||
# Move the CPU slot_mapping to GPU on the copy stream so the
|
||||
# implicit H2D inside fancy indexing doesn't sync the default
|
||||
# stream.
|
||||
slot_mapping_gpu = slot_mapping.to(
|
||||
device=self._kv_cache.device, non_blocking=True
|
||||
)
|
||||
# stream. Deliberate bulk transfer in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
slot_mapping_gpu = slot_mapping.to(
|
||||
device=self._kv_cache.device, non_blocking=True
|
||||
)
|
||||
hidden_states_gpu = extract_from_kv_cache(
|
||||
self._kv_cache, slot_mapping_gpu, num_tokens
|
||||
)
|
||||
|
||||
@@ -265,6 +265,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down, round_up
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
@@ -2510,8 +2511,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu[reqs_start + 1 : num_reqs + 1]
|
||||
- query_start_loc_cpu[reqs_start:num_reqs]
|
||||
)
|
||||
context_lens_cpu = (
|
||||
seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu
|
||||
context_lens_cpu = torch.empty(
|
||||
num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY
|
||||
)
|
||||
torch.subtract(
|
||||
seq_lens_cpu[reqs_start:num_reqs],
|
||||
prefill_query_lens_cpu,
|
||||
out=context_lens_cpu,
|
||||
)
|
||||
prefill_query_start_loc = (
|
||||
query_start_loc[reqs_start:] - query_start_loc[reqs_start]
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.worker.ubatching import (
|
||||
dbo_enabled,
|
||||
dbo_maybe_run_recv_hook,
|
||||
@@ -109,7 +110,10 @@ class ExpertTokensMetadata:
|
||||
expert_num_tokens_list: list[int], device: str
|
||||
) -> "ExpertTokensMetadata":
|
||||
expert_num_tokens_cpu = torch.tensor(
|
||||
expert_num_tokens_list, device="cpu", dtype=torch.int32
|
||||
expert_num_tokens_list,
|
||||
device="cpu",
|
||||
dtype=torch.int32,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
return ExpertTokensMetadata(
|
||||
expert_num_tokens=expert_num_tokens_cpu.to(device, non_blocking=True),
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -80,10 +81,7 @@ class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer):
|
||||
dim=0,
|
||||
)
|
||||
lengths_cpu = spatial_shapes.prod(dim=-1).to(torch.int32)
|
||||
lengths = lengths_cpu.to(
|
||||
device=pixel_values.device,
|
||||
non_blocking=True,
|
||||
)
|
||||
lengths = async_tensor_h2d(lengths_cpu, pixel_values.device)
|
||||
|
||||
cu_seqlens = torch.zeros(
|
||||
lengths.numel() + 1,
|
||||
|
||||
@@ -538,7 +538,7 @@ class Ernie4_5_VisionTransformer(nn.Module):
|
||||
else:
|
||||
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
|
||||
|
||||
cu_seqlens = cu_seqlens.to(device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, device)
|
||||
|
||||
return {
|
||||
"rotary_pos_emb": rotary_pos_emb,
|
||||
@@ -1746,8 +1746,8 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
# Eager fallback: run the full pipeline (ViT + resampler). The result
|
||||
# is scattered directly, so it must be the post-merge embeddings.
|
||||
pixel_values = mm_kwargs["pixel_values"].type(self.vision_model.dtype)
|
||||
grid_thw = mm_kwargs["image_grid_thw"].to(
|
||||
self.vision_model.device, non_blocking=True
|
||||
grid_thw = async_tensor_h2d(
|
||||
mm_kwargs["image_grid_thw"], self.vision_model.device
|
||||
)
|
||||
image_features = self.vision_model(pixel_values, grid_thw)
|
||||
return self.resampler_model(image_features, grid_thw)
|
||||
@@ -1768,7 +1768,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
output = outputs["default"]
|
||||
assert batch_mm_kwargs is not None
|
||||
grid_thw_cpu = batch_mm_kwargs["image_grid_thw"]
|
||||
grid_thw = grid_thw_cpu.to(output.device, non_blocking=True)
|
||||
grid_thw = async_tensor_h2d(grid_thw_cpu, output.device)
|
||||
# The valid token count slices the graph output for the eager
|
||||
# resampler call, so it has to come back to the host.
|
||||
num_valid = int(
|
||||
|
||||
@@ -70,6 +70,7 @@ from vllm.model_executor.models.glm4_1v import (
|
||||
Glm4vVisionTransformer,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
from .utils import (
|
||||
maybe_prefix,
|
||||
@@ -344,8 +345,11 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
|
||||
cu_seqlens = torch.repeat_interleave(
|
||||
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
|
||||
).cumsum(dim=0, dtype=torch.int32)
|
||||
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
|
||||
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
|
||||
# cat straight into a pinned buffer so the H2D copy stays non-blocking.
|
||||
cu_seqlens = torch.cat(
|
||||
[cu_seqlens.new_zeros(1), cu_seqlens],
|
||||
out=cu_seqlens.new_empty(cu_seqlens.numel() + 1, pin_memory=PIN_MEMORY),
|
||||
).to(self.device, non_blocking=True)
|
||||
|
||||
# pre-compute max_seqlen for attn mask to reduce cuMemcpy operations
|
||||
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
|
||||
|
||||
@@ -69,6 +69,7 @@ from vllm.transformers_utils.processors.isaac import (
|
||||
get_image_size_for_max_num_patches,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
|
||||
from .vision import is_vit_use_data_parallel
|
||||
|
||||
@@ -970,9 +971,14 @@ class IsaacForConditionalGeneration(
|
||||
device = next(self.language_model.parameters()).device
|
||||
dtype = self.vision_embedding.linear_fc1.weight.dtype
|
||||
pixel_values = pixel_values.to(device=device, dtype=dtype)
|
||||
spatial_grids = image_grid_thw[:, 1:3].to(
|
||||
device, dtype=torch.int32, non_blocking=True
|
||||
)
|
||||
# The [:, 1:3] column slice isn't densely laid out, so stage it with a
|
||||
# single copy straight into pinned memory to keep the H2D non-blocking.
|
||||
spatial_grids_cpu = torch.empty(
|
||||
(image_grid_thw.shape[0], 2),
|
||||
dtype=image_grid_thw.dtype,
|
||||
pin_memory=PIN_MEMORY,
|
||||
).copy_(image_grid_thw[:, 1:3])
|
||||
spatial_grids = async_tensor_h2d(spatial_grids_cpu, device, dtype=torch.int32)
|
||||
|
||||
vision_embeddings = self.vision_embedding((pixel_values, spatial_grids))
|
||||
merge_size = self.config.vision_config.pixel_shuffle_scale_factor
|
||||
|
||||
@@ -59,7 +59,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -1305,17 +1305,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
||||
)
|
||||
else:
|
||||
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
|
||||
# These are all built on the host, so stage them over
|
||||
# non-blocking rather than forcing a sync per transfer.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
# These are all built on the host; concat straight into pinned
|
||||
# buffers so the H2D copies stay non-blocking.
|
||||
siglip_position_ids = torch.concat(
|
||||
siglip_position_ids,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in siglip_position_ids),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values.device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values.device
|
||||
)
|
||||
sample_indices = torch.concat(sample_indices, dim=0).to(
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
sample_indices = torch.concat(
|
||||
sample_indices,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in sample_indices),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values.device, non_blocking=True)
|
||||
|
||||
image_embeds = self.visual(
|
||||
pixel_values=pixel_values,
|
||||
@@ -1359,16 +1371,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
||||
)
|
||||
else:
|
||||
pixel_values_videos = pixel_values_videos.type(self.visual.dtype)
|
||||
# These are host-built; stage them over non-blocking.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values_videos.device, non_blocking=True
|
||||
)
|
||||
# These are host-built; concat straight into pinned buffers so
|
||||
# the H2D copies stay non-blocking.
|
||||
siglip_position_ids = torch.concat(
|
||||
siglip_position_ids,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in siglip_position_ids),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values_videos.device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values_videos.device
|
||||
)
|
||||
sample_indices = torch.concat(sample_indices, dim=0).to(
|
||||
pixel_values_videos.device, non_blocking=True
|
||||
)
|
||||
sample_indices = torch.concat(
|
||||
sample_indices,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in sample_indices),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values_videos.device, non_blocking=True)
|
||||
|
||||
video_embeds = self.visual(
|
||||
pixel_values=pixel_values_videos,
|
||||
|
||||
@@ -68,7 +68,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .ernie45 import Ernie4_5ForCausalLM
|
||||
@@ -1142,10 +1142,17 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
|
||||
siglip_position_ids.append(image_position_ids)
|
||||
cu_seqlens.append(cu_seqlens[-1] + numel)
|
||||
|
||||
# Both are built on the host; stage them over non-blocking.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
# Both are built on the host; concat straight into a pinned buffer
|
||||
# so the H2D copy stays non-blocking.
|
||||
siglip_position_ids = torch.concat(
|
||||
siglip_position_ids,
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in siglip_position_ids),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
).to(pixel_values.device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values.device
|
||||
)
|
||||
|
||||
@@ -643,8 +643,8 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
seq_len, batch_size, chunk_size_nc, left_chunk_nc
|
||||
)
|
||||
if device.type != "cpu":
|
||||
enc_streaming_mask_nc = enc_streaming_mask_nc.contiguous().to(
|
||||
device, non_blocking=True
|
||||
enc_streaming_mask_nc = async_tensor_h2d(
|
||||
enc_streaming_mask_nc.contiguous(), device
|
||||
)
|
||||
if masks is not None:
|
||||
hs_mask_nc = masks & enc_streaming_mask_nc
|
||||
|
||||
@@ -91,6 +91,7 @@ from vllm.multimodal.processing.processor import (
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -985,8 +986,8 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
input_features = audio_input["input_features"]
|
||||
# audio_feature_lengths is keep_on_cpu; the audio tower derives
|
||||
# device placement from feature_lens, so move it explicitly.
|
||||
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
|
||||
input_features.device, non_blocking=True
|
||||
audio_feature_lengths = async_tensor_h2d(
|
||||
audio_input["audio_feature_lengths"], input_features.device
|
||||
)
|
||||
|
||||
audio_feat_lengths, audio_output_lengths = (
|
||||
|
||||
@@ -1001,7 +1001,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
|
||||
|
||||
# Move cu_seqlens to GPU; grid_thw may be on CPU during profile_run
|
||||
# and FA3 vit attention requires cu_seqlens on CUDA.
|
||||
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, self.device)
|
||||
hidden_states = hidden_states.unsqueeze(1)
|
||||
rotary_pos_emb_cos = rotary_pos_emb_cos.to(hidden_states.device)
|
||||
rotary_pos_emb_sin = rotary_pos_emb_sin.to(hidden_states.device)
|
||||
@@ -1574,8 +1574,8 @@ class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMix
|
||||
input_features = audio_input["input_features"]
|
||||
# audio_feature_lengths is keep_on_cpu; the audio tower derives
|
||||
# device placement from feature_lens, so move it explicitly.
|
||||
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
|
||||
input_features.device, non_blocking=True
|
||||
audio_feature_lengths = async_tensor_h2d(
|
||||
audio_input["audio_feature_lengths"], input_features.device
|
||||
)
|
||||
|
||||
audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)
|
||||
|
||||
@@ -2367,8 +2367,8 @@ class Qwen3VLForConditionalGeneration(
|
||||
grid_thw_list = grid_thw.tolist()
|
||||
image_embeds_out = []
|
||||
for emb, size in zip(image_embeds_split, grid_thw_list):
|
||||
positions = compute_mrope_for_media(size, merge_size).to(
|
||||
emb.device, non_blocking=True
|
||||
positions = async_tensor_h2d(
|
||||
compute_mrope_for_media(size, merge_size), emb.device
|
||||
)
|
||||
positions = torch.cat(
|
||||
[
|
||||
@@ -2596,14 +2596,11 @@ class Qwen3VLForConditionalGeneration(
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=0, length=len(unpruned_token_ids)),
|
||||
)
|
||||
original_mrope = (
|
||||
self.get_mrope_input_positions(
|
||||
input_tokens=unpruned_token_ids,
|
||||
mm_features=[mm_feature],
|
||||
)[0]
|
||||
.to(device, non_blocking=True)
|
||||
.permute(1, 0)
|
||||
)
|
||||
original_mrope_cpu = self.get_mrope_input_positions(
|
||||
input_tokens=unpruned_token_ids,
|
||||
mm_features=[mm_feature],
|
||||
)[0]
|
||||
original_mrope = async_tensor_h2d(original_mrope_cpu, device).permute(1, 0)
|
||||
full_is_video_embed = unpruned_token_ids_tensor == embed_token_id
|
||||
|
||||
with gpu_sync_allowed():
|
||||
|
||||
@@ -21,6 +21,7 @@ import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -181,6 +182,31 @@ def _lazy_import_wrapper(
|
||||
return wrapper
|
||||
|
||||
|
||||
def pin_host_range_buf(length: int) -> None:
|
||||
"""Pin flashinfer's cached host-side arange buffer covering `length`.
|
||||
|
||||
flashinfer's decode plan() builds its host qo_indptr from the cached
|
||||
`_get_range_buf` entry and copies it to the GPU with non_blocking=True on
|
||||
every call; the cache entry is unpinned, so the copy silently blocks the
|
||||
host. Replace the entry with a pinned copy so the copies stay async.
|
||||
"""
|
||||
if not PIN_MEMORY:
|
||||
return
|
||||
mod = _get_submodule("flashinfer.utils")
|
||||
if mod is None:
|
||||
return
|
||||
get_range_buf = getattr(mod, "_get_range_buf", None)
|
||||
cache_buf = getattr(mod, "_cache_buf", None)
|
||||
ceil_pow2 = getattr(mod, "_ceil_pow2", None)
|
||||
if get_range_buf is None or cache_buf is None or ceil_pow2 is None:
|
||||
return
|
||||
get_range_buf(length, "cpu") # Ensure the cache entry exists.
|
||||
key = (f"range_{ceil_pow2(length)}", "cpu")
|
||||
buf = cache_buf[key]
|
||||
if not buf.is_pinned():
|
||||
cache_buf[key] = buf.pin_memory()
|
||||
|
||||
|
||||
# Create lazy wrappers for each function
|
||||
flashinfer_trtllm_bf16_moe = _lazy_import_wrapper(
|
||||
"flashinfer.fused_moe", "trtllm_bf16_moe"
|
||||
|
||||
@@ -15,6 +15,7 @@ from flashinfer import (
|
||||
BatchPrefillWithPagedKVCacheWrapper,
|
||||
BatchPrefillWithRaggedKVCacheWrapper,
|
||||
MultiLevelCascadeAttentionWrapper,
|
||||
get_seq_lens,
|
||||
)
|
||||
from flashinfer.decode import fast_decode_plan, trtllm_batch_decode_with_kv_cache
|
||||
from flashinfer.prefill import trtllm_batch_context_with_kv_cache
|
||||
@@ -43,6 +44,7 @@ from vllm.utils.flashinfer import (
|
||||
can_use_trtllm_attention,
|
||||
flashinfer_xqa_batch_decode_with_kv_cache,
|
||||
force_use_trtllm_attention,
|
||||
pin_host_range_buf,
|
||||
supports_trtllm_attention,
|
||||
use_trtllm_attention,
|
||||
)
|
||||
@@ -299,6 +301,7 @@ class BatchDCPPrefillWrapper:
|
||||
paged_kv_indptr_cpu: torch.Tensor,
|
||||
paged_kv_indices: torch.Tensor,
|
||||
paged_kv_last_page_len_cpu: torch.Tensor,
|
||||
kv_lens_cpu: torch.Tensor,
|
||||
page_size: int,
|
||||
num_qo_heads: int,
|
||||
dcp_world_size: int,
|
||||
@@ -318,6 +321,7 @@ class BatchDCPPrefillWrapper:
|
||||
paged_kv_indptr=paged_kv_indptr_cpu,
|
||||
paged_kv_indices=paged_kv_indices,
|
||||
paged_kv_last_page_len=paged_kv_last_page_len_cpu,
|
||||
seq_lens=kv_lens_cpu,
|
||||
num_qo_heads=num_qo_heads * dcp_world_size,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim_qk=head_dim,
|
||||
@@ -1580,6 +1584,17 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
paged_kv_last_page_len_prefill_cpu = (
|
||||
paged_kv_last_page_len_prefill_cpu.pin_memory()
|
||||
)
|
||||
# plan() also copies kv_lens to the GPU with non_blocking=True,
|
||||
# deriving them via get_seq_lens() into unpinned memory when
|
||||
# seq_lens is not passed; pass an explicit pinned copy of the
|
||||
# same values so that copy stays async too.
|
||||
kv_lens_prefill_cpu = get_seq_lens(
|
||||
paged_kv_indptr_prefill_cpu,
|
||||
paged_kv_last_page_len_prefill_cpu,
|
||||
page_size,
|
||||
)
|
||||
if PIN_MEMORY:
|
||||
kv_lens_prefill_cpu = kv_lens_prefill_cpu.pin_memory()
|
||||
if self.use_dcp:
|
||||
assert isinstance(prefill_wrapper, BatchDCPPrefillWrapper)
|
||||
prefill_wrapper.plan(
|
||||
@@ -1587,6 +1602,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
paged_kv_indptr_cpu=paged_kv_indptr_prefill_cpu,
|
||||
paged_kv_indices=paged_kv_indices,
|
||||
paged_kv_last_page_len_cpu=paged_kv_last_page_len_prefill_cpu,
|
||||
kv_lens_cpu=kv_lens_prefill_cpu,
|
||||
page_size=self.page_size,
|
||||
num_qo_heads=self.num_qo_heads,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
@@ -1616,6 +1632,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
paged_kv_indptr=paged_kv_indptr_prefill_cpu,
|
||||
paged_kv_indices=paged_kv_indices,
|
||||
paged_kv_last_page_len=paged_kv_last_page_len_prefill_cpu,
|
||||
seq_lens=kv_lens_prefill_cpu,
|
||||
num_qo_heads=self.num_qo_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_dim_qk=self.head_dim,
|
||||
@@ -1716,11 +1733,22 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
if PIN_MEMORY:
|
||||
paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory()
|
||||
paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory()
|
||||
# plan() also copies kv_lens to the GPU with non_blocking=True
|
||||
# (trtllm-gen/cute-dsl backends), deriving them via
|
||||
# get_seq_lens() into unpinned memory when seq_lens is not
|
||||
# passed; pass an explicit pinned copy of the same values
|
||||
# so that copy stays async too.
|
||||
kv_lens_decode_cpu = get_seq_lens(
|
||||
paged_kv_indptr_cpu, paged_kv_last_page_len_cpu, page_size
|
||||
)
|
||||
if PIN_MEMORY:
|
||||
kv_lens_decode_cpu = kv_lens_decode_cpu.pin_memory()
|
||||
fast_plan_decode(
|
||||
decode_wrapper,
|
||||
indptr_cpu=paged_kv_indptr_cpu,
|
||||
indices=paged_kv_indices,
|
||||
last_page_len_cpu=paged_kv_last_page_len_cpu,
|
||||
seq_lens_cpu=kv_lens_decode_cpu,
|
||||
num_qo_heads=self.num_qo_heads * self.dcp_world_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_dim=self.head_dim,
|
||||
@@ -2579,6 +2607,7 @@ def fast_plan_decode(
|
||||
indptr_cpu: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
last_page_len_cpu: torch.Tensor,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
num_qo_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
@@ -2614,6 +2643,10 @@ def fast_plan_decode(
|
||||
# original plan if we run for dynamic shape. For fixed shape (cudagraph),
|
||||
# this warm up is to generate the _cached_module for the decode wrapper.
|
||||
if not self.is_cuda_graph_enabled or getattr(self, "vllm_first_call", True):
|
||||
# plan() copies qo_indptr from flashinfer's internal cached host
|
||||
# arange buffer (_get_range_buf) with non_blocking=True; pin the
|
||||
# cache entry so the copy stays async.
|
||||
pin_host_range_buf(len(last_page_len_cpu) + 1)
|
||||
self.plan(
|
||||
indptr=indptr_cpu,
|
||||
indices=indices,
|
||||
@@ -2634,7 +2667,7 @@ def fast_plan_decode(
|
||||
rope_theta=rope_theta,
|
||||
non_blocking=non_blocking,
|
||||
block_tables=None,
|
||||
seq_lens=None,
|
||||
seq_lens=seq_lens_cpu,
|
||||
fixed_split_size=fixed_split_size,
|
||||
disable_split_kv=disable_split_kv,
|
||||
)
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -216,8 +217,13 @@ class HpcAttnMetadataBuilder(AttentionMetadataBuilder[HpcAttnMetadata]):
|
||||
if num_prefills > 0:
|
||||
qo_indptr_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
prefill_start = num_decodes
|
||||
qo_indptr_prefill_cpu = (
|
||||
qo_indptr_cpu[prefill_start:] - qo_indptr_cpu[prefill_start]
|
||||
qo_indptr_prefill_cpu = torch.empty(
|
||||
num_prefills + 1, dtype=torch.int32, pin_memory=PIN_MEMORY
|
||||
)
|
||||
torch.subtract(
|
||||
qo_indptr_cpu[prefill_start:],
|
||||
qo_indptr_cpu[prefill_start],
|
||||
out=qo_indptr_prefill_cpu,
|
||||
)
|
||||
qo_indptr = qo_indptr_prefill_cpu.to(self.device, non_blocking=True)
|
||||
|
||||
|
||||
@@ -21,7 +21,11 @@ from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import get_dtype_size, is_quantized_kv_cache
|
||||
from vllm.utils.torch_utils import (
|
||||
async_tensor_h2d,
|
||||
get_dtype_size,
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -593,6 +597,7 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
num_extends + 1,
|
||||
dtype=torch.int32,
|
||||
device=seq_lens_for_extend.device,
|
||||
pin_memory=True,
|
||||
)
|
||||
torch.cumsum(
|
||||
swa_seqlen_for_extend,
|
||||
@@ -623,8 +628,8 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
|
||||
swa_metadata = AiterChunkSlidingWindowMetadata(
|
||||
swa_cu_seqlens=cu_seq_lens.to(self.device, non_blocking=True),
|
||||
swa_seq_starts=seq_starts.to(self.device, non_blocking=True),
|
||||
swa_token_to_batch=token_to_seq.to(self.device, non_blocking=True),
|
||||
swa_seq_starts=async_tensor_h2d(seq_starts, self.device),
|
||||
swa_token_to_batch=async_tensor_h2d(token_to_seq, self.device),
|
||||
swa_max_seqlens=max_seqlen_k,
|
||||
swa_total_tokens=total_tokens,
|
||||
swa_workspace=swa_workspace,
|
||||
@@ -668,8 +673,8 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
chunk_context_metadata = AiterChunkContextMetadata(
|
||||
workspace=self.extend_workspace,
|
||||
cu_seq_lens_chunk=cu_seq_lens_cpu.to(self.device, non_blocking=True),
|
||||
chunk_starts=chunk_starts.to(self.device, non_blocking=True),
|
||||
token_to_batch=token_to_batch_tensor.to(self.device, non_blocking=True),
|
||||
chunk_starts=async_tensor_h2d(chunk_starts, self.device),
|
||||
token_to_batch=async_tensor_h2d(token_to_batch_tensor, self.device),
|
||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||
num_chunks=num_chunks,
|
||||
total_token_per_batch=cu_seq_lens_cpu[:, -1].tolist(),
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.config import ParallelConfig
|
||||
from vllm.distributed.parallel_state import get_dp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.worker.ubatch_utils import (
|
||||
check_ubatch_thresholds,
|
||||
is_last_ubatch_empty,
|
||||
@@ -47,7 +48,8 @@ def _run_ar(
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
device, group = _get_device_and_group(parallel_config)
|
||||
# Populate this rank's contribution on CPU to reduce GPU syncs.
|
||||
tensor_cpu = torch.zeros(4, dp_size, dtype=torch.int32)
|
||||
pin_memory = PIN_MEMORY and torch.device(device).type == "cuda"
|
||||
tensor_cpu = torch.zeros(4, dp_size, dtype=torch.int32, pin_memory=pin_memory)
|
||||
tensor_cpu[0][dp_rank] = orig_num_tokens_per_ubatch
|
||||
tensor_cpu[1][dp_rank] = padded_num_tokens_per_ubatch
|
||||
tensor_cpu[2][dp_rank] = 1 if should_ubatch else 0
|
||||
|
||||
@@ -2258,8 +2258,10 @@ class GPUModelRunner(
|
||||
if self.use_async_spec_decode and (self.uses_mrope or self.uses_xdrope_dim > 0):
|
||||
drift = self.num_computed_tokens[req_indices_gpu].to(
|
||||
torch.int64
|
||||
) - self.input_batch.num_computed_tokens_cpu_tensor[req_indices].to(
|
||||
device=self.device, dtype=torch.int64, non_blocking=True
|
||||
) - async_tensor_h2d(
|
||||
self.input_batch.num_computed_tokens_cpu_tensor[req_indices],
|
||||
device=self.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
target = self.mrope_positions if self.uses_mrope else self.xdrope_positions
|
||||
target.gpu[:, :total_num_scheduled_tokens] += drift
|
||||
@@ -6324,9 +6326,7 @@ class GPUModelRunner(
|
||||
self.eplb_step(is_dummy=True, is_profile=is_profile)
|
||||
|
||||
logit_indices = np.cumsum(num_scheduled_tokens) - 1
|
||||
logit_indices_device = torch.from_numpy(logit_indices).to(
|
||||
self.device, non_blocking=True
|
||||
)
|
||||
logit_indices_device = async_tensor_h2d(logit_indices, device=self.device)
|
||||
return hidden_states, hidden_states[logit_indices_device]
|
||||
|
||||
@torch.inference_mode()
|
||||
|
||||
Reference in New Issue
Block a user