[Perf] Avoid more h2d copies from non-pinned tensors (#54660)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3ba9907a1d
commit
a566ea7e8f
@@ -805,7 +805,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
window_index_thw, cu_seqlens_window_thw = self.get_window_index_thw(t, h, w)
|
||||
cos_thw, sin_thw = self.rotary_pos_emb_thw(t, h, w)
|
||||
|
||||
window_index_thw_dev = window_index_thw.to(cos_thw.device, non_blocking=True)
|
||||
window_index_thw_dev = async_tensor_h2d(window_index_thw, cos_thw.device)
|
||||
cos_thw = cos_thw[window_index_thw_dev, :, :]
|
||||
cos_thw = cos_thw.flatten(start_dim=0, end_dim=1)
|
||||
sin_thw = sin_thw[window_index_thw_dev, :, :]
|
||||
@@ -1053,7 +1053,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
)
|
||||
rotary_pos_emb_cos = rotary_pos_emb_cos.to(device=device, non_blocking=True)
|
||||
rotary_pos_emb_sin = rotary_pos_emb_sin.to(device=device, non_blocking=True)
|
||||
window_index = window_index.to(device=device, non_blocking=True)
|
||||
window_index = async_tensor_h2d(window_index, device)
|
||||
reverse_indices = reverse_indices.to(device=device, non_blocking=True)
|
||||
|
||||
metadata["rotary_pos_emb_cos"] = rotary_pos_emb_cos
|
||||
@@ -1475,8 +1475,8 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
grid_thw_list = grid_thw.tolist()
|
||||
image_embeds_out = []
|
||||
for emb, size in zip(image_embeds_split, grid_thw_list):
|
||||
positions = compute_mrope_for_media(size, merge_size).to(
|
||||
emb.device, non_blocking=True
|
||||
positions = async_tensor_h2d(
|
||||
compute_mrope_for_media(size, merge_size), emb.device
|
||||
)
|
||||
emb = torch.cat([emb, positions], dim=1)
|
||||
image_embeds_out.append(emb)
|
||||
@@ -1558,12 +1558,13 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
spatial_merge_size=self.visual.spatial_merge_size,
|
||||
q=self.video_pruning_rate,
|
||||
)
|
||||
positions = compute_mrope_for_media(
|
||||
positions_cpu = compute_mrope_for_media(
|
||||
size,
|
||||
merge_size,
|
||||
tokens_per_second=tokens_per_second,
|
||||
video_second_per_grid=video_second_per_grid_t.item(),
|
||||
).to(emb.device, non_blocking=True)
|
||||
)
|
||||
positions = async_tensor_h2d(positions_cpu, emb.device)
|
||||
|
||||
with gpu_sync_allowed():
|
||||
emb = emb[retention_mask]
|
||||
|
||||
@@ -85,6 +85,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||
|
||||
@@ -707,7 +708,7 @@ class Qwen2VisionTransformer(nn.Module):
|
||||
return {
|
||||
"rotary_pos_emb_cos": rotary_pos_emb_cos,
|
||||
"rotary_pos_emb_sin": rotary_pos_emb_sin,
|
||||
"cu_seqlens": cu_seqlens.to(device=device, non_blocking=True),
|
||||
"cu_seqlens": async_tensor_h2d(cu_seqlens, device),
|
||||
"max_seqlen": max_seqlen,
|
||||
}
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import BatchFeature, PreTrainedModel
|
||||
@@ -1180,8 +1181,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
# grid_thw fields are registered keep_on_cpu; restore the on-device
|
||||
# placement that HF get_image_features implementations expect.
|
||||
for key, value in kwargs.items():
|
||||
if isinstance(value, torch.Tensor):
|
||||
kwargs[key] = value.to(pixel_values.device, non_blocking=True)
|
||||
if isinstance(value, torch.Tensor) and value.is_cpu:
|
||||
kwargs[key] = async_tensor_h2d(value, pixel_values.device)
|
||||
|
||||
# The underlying HuggingFace `get_image_features` implementations
|
||||
# contain model-internal syncs (e.g. Idefics3 filters all-zero
|
||||
|
||||
+37
-51
@@ -293,21 +293,39 @@ def _nested_tensors_are_cpu(tensors: NestedTensors) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _is_dense(t: torch.Tensor) -> bool:
|
||||
"""Whether `t`'s elements fill a contiguous storage range, possibly
|
||||
permuted (e.g. a transposed matrix). Such layouts can be copied with
|
||||
pitched cudaMemcpy2D/3DAsync directly from pinned memory, whereas
|
||||
gapped layouts (e.g. strided slices) stage through a pageable temp."""
|
||||
if not t.is_contiguous():
|
||||
expected_stride = 1
|
||||
for stride, size in sorted(zip(t.stride(), t.shape)):
|
||||
if size > 1:
|
||||
if stride != expected_stride:
|
||||
return False
|
||||
expected_stride *= size
|
||||
return True
|
||||
|
||||
|
||||
def _nested_tensors_h2d(
|
||||
tensors: NestedTensors,
|
||||
device: torch.types.Device,
|
||||
*,
|
||||
pin_memory: bool = False,
|
||||
) -> NestedTensors:
|
||||
if device is None:
|
||||
return tensors
|
||||
|
||||
return json_map_leaves(
|
||||
(
|
||||
lambda x: x.to(device=device, non_blocking=True)
|
||||
if isinstance(x, torch.Tensor)
|
||||
else x
|
||||
),
|
||||
tensors,
|
||||
)
|
||||
def _h2d(x: torch.Tensor) -> torch.Tensor:
|
||||
if not isinstance(x, torch.Tensor):
|
||||
return x
|
||||
if pin_memory and x.is_cpu and not (x.is_pinned() and _is_dense(x)):
|
||||
# Ensure tensor is pinned and dense for non_blocking H2D copy.
|
||||
x = x.new_empty(x.shape, pin_memory=True).copy_(x)
|
||||
return x.to(device=device, non_blocking=True)
|
||||
|
||||
return json_map_leaves(_h2d, tensors)
|
||||
|
||||
|
||||
BatchedTensorInputs: TypeAlias = dict[str, NestedTensors]
|
||||
@@ -452,10 +470,7 @@ class BaseMultiModalField(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def _reduce_data(
|
||||
self,
|
||||
batch: list[NestedTensors],
|
||||
*,
|
||||
pin_memory: bool,
|
||||
self, batch: list[NestedTensors], *, pin_memory: bool
|
||||
) -> NestedTensors:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -491,7 +506,7 @@ class BaseMultiModalField(ABC):
|
||||
# concept, and `pin_memory()` rejects device tensors outright.
|
||||
pin_memory = False
|
||||
out = self._reduce_data(batch, pin_memory=pin_memory)
|
||||
return _nested_tensors_h2d(out, device=device)
|
||||
return _nested_tensors_h2d(out, device=device, pin_memory=pin_memory)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
@@ -511,10 +526,7 @@ class MultiModalBatchedField(BaseMultiModalField):
|
||||
return [field_factory(item) for item in data]
|
||||
|
||||
def _reduce_data(
|
||||
self,
|
||||
batch: list[NestedTensors],
|
||||
*,
|
||||
pin_memory: bool,
|
||||
self, batch: list[NestedTensors], *, pin_memory: bool
|
||||
) -> NestedTensors:
|
||||
if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
|
||||
batch = cast(list[torch.Tensor], batch)
|
||||
@@ -522,20 +534,11 @@ class MultiModalBatchedField(BaseMultiModalField):
|
||||
# An optimization when `batch` contains only one tensor:
|
||||
# - produce exactly same result as `torch.stack(batch)`
|
||||
# - will achieve zero-copy if the tensor is contiguous
|
||||
out = batch[0].unsqueeze(0)
|
||||
if not pin_memory:
|
||||
return out.contiguous()
|
||||
# Avoid extra copy - pinning unpinned memory will make it contiguous
|
||||
if not out.is_contiguous() and out.is_pinned():
|
||||
out = out.contiguous()
|
||||
return out.pin_memory()
|
||||
return batch[0].unsqueeze(0)
|
||||
first_shape = batch[0].shape
|
||||
if all(elem.shape == first_shape for elem in batch):
|
||||
out = torch.empty(
|
||||
(len(batch), *batch[0].shape),
|
||||
dtype=batch[0].dtype,
|
||||
device=batch[0].device,
|
||||
pin_memory=pin_memory,
|
||||
out = batch[0].new_empty(
|
||||
(len(batch), *first_shape), pin_memory=pin_memory
|
||||
)
|
||||
return torch.stack(batch, out=out)
|
||||
|
||||
@@ -567,10 +570,7 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
return [field_factory(data[cast(slice, s)]) for s in self.slices]
|
||||
|
||||
def _reduce_data(
|
||||
self,
|
||||
batch: list[NestedTensors],
|
||||
*,
|
||||
pin_memory: bool,
|
||||
self, batch: list[NestedTensors], *, pin_memory: bool
|
||||
) -> NestedTensors:
|
||||
if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
|
||||
batch = cast(list[torch.Tensor], batch)
|
||||
@@ -578,13 +578,7 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
# An optimization when `batch` contains only one tensor:
|
||||
# - produce exactly same result as `torch.concat(batch)`
|
||||
# - will achieve zero-copy if the tensor is contiguous
|
||||
out = batch[0]
|
||||
if not pin_memory:
|
||||
return out.contiguous()
|
||||
# Avoid extra copy - pinning unpinned memory will make it contiguous
|
||||
if not out.is_contiguous() and out.is_pinned():
|
||||
out = out.contiguous()
|
||||
return out.pin_memory()
|
||||
return batch[0]
|
||||
|
||||
dim = self.dim + (self.dim < 0) * len(batch[0].shape)
|
||||
|
||||
@@ -596,11 +590,8 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
if all(_shape_before_after(elem) == first_shape for elem in batch):
|
||||
shape_before, shape_after = first_shape
|
||||
shape_concat = sum(item.shape[dim] for item in batch)
|
||||
out = torch.empty(
|
||||
(*shape_before, shape_concat, *shape_after),
|
||||
dtype=batch[0].dtype,
|
||||
device=batch[0].device,
|
||||
pin_memory=pin_memory,
|
||||
out = batch[0].new_empty(
|
||||
(*shape_before, shape_concat, *shape_after), pin_memory=pin_memory
|
||||
)
|
||||
return torch.concat(batch, dim=self.dim, out=out)
|
||||
|
||||
@@ -622,12 +613,7 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
max_sizes.append(max(t.shape[d] for t in batch))
|
||||
|
||||
# Step 2: Create zero-initialized output tensor
|
||||
out = torch.zeros(
|
||||
max_sizes,
|
||||
dtype=batch[0].dtype,
|
||||
device=batch[0].device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
out = batch[0].new_zeros(max_sizes, pin_memory=pin_memory)
|
||||
|
||||
# Step 3: Slice-assign each tensor to its proper position
|
||||
concat_offset = 0
|
||||
|
||||
@@ -1569,6 +1569,17 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
prefill_start : num_reqs + 1
|
||||
]
|
||||
assert paged_kv_indptr_prefill_cpu.shape[0] == num_prefills + 1
|
||||
# plan() copies these to the GPU with non_blocking=True;
|
||||
# stage them in pinned memory so the copies stay async
|
||||
# (the reused buffers themselves are intentionally not pinned).
|
||||
if PIN_MEMORY:
|
||||
qo_indptr_prefill_cpu = qo_indptr_prefill_cpu.pin_memory()
|
||||
paged_kv_indptr_prefill_cpu = (
|
||||
paged_kv_indptr_prefill_cpu.pin_memory()
|
||||
)
|
||||
paged_kv_last_page_len_prefill_cpu = (
|
||||
paged_kv_last_page_len_prefill_cpu.pin_memory()
|
||||
)
|
||||
if self.use_dcp:
|
||||
assert isinstance(prefill_wrapper, BatchDCPPrefillWrapper)
|
||||
prefill_wrapper.plan(
|
||||
@@ -1695,13 +1706,21 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
o_dtype = (
|
||||
FP8_DTYPE if self.is_kvcache_nvfp4 else self.model_config.dtype
|
||||
)
|
||||
paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_input_tokens + 1]
|
||||
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[
|
||||
:num_input_tokens
|
||||
]
|
||||
# plan() copies these to the GPU with non_blocking=True;
|
||||
# stage them in pinned memory so the copies stay async
|
||||
# (the reused buffers themselves are intentionally not pinned).
|
||||
if PIN_MEMORY:
|
||||
paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory()
|
||||
paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory()
|
||||
fast_plan_decode(
|
||||
decode_wrapper,
|
||||
indptr_cpu=self.paged_kv_indptr.cpu[: num_input_tokens + 1],
|
||||
indptr_cpu=paged_kv_indptr_cpu,
|
||||
indices=paged_kv_indices,
|
||||
last_page_len_cpu=self.paged_kv_last_page_len.cpu[
|
||||
:num_input_tokens
|
||||
],
|
||||
last_page_len_cpu=paged_kv_last_page_len_cpu,
|
||||
num_qo_heads=self.num_qo_heads * self.dcp_world_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_dim=self.head_dim,
|
||||
|
||||
Reference in New Issue
Block a user