[Perf] Avoid more GPU<->CPU syncs in multimodal encoders (#52369)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -68,6 +68,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .ernie45 import Ernie4_5ForCausalLM
|
||||
@@ -800,16 +801,18 @@ class SiglipEncoder(nn.Module):
|
||||
[height_position_ids, width_position_ids],
|
||||
dim=-1,
|
||||
)
|
||||
max_grid_size = pids.max() + 1
|
||||
# The ids are built from the grids above, so `h`/`w` bound them
|
||||
# and the table size is known on the host.
|
||||
max_grid_size = max(max(h, w) for _, h, w in flatten_image_grid_thw)
|
||||
rope_emb_max_grid = self.rotary_pos_emb(max_grid_size)
|
||||
rotary_pos_emb = rope_emb_max_grid[pids].flatten(1)
|
||||
|
||||
if cu_seqlens is None:
|
||||
raise ValueError("cu_seqlens cannot be None for SiglipEncoder.")
|
||||
if not isinstance(cu_seqlens, torch.Tensor):
|
||||
cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32, device=device)
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, dtype=torch.int32, device=device)
|
||||
else:
|
||||
cu_seqlens = cu_seqlens.to(device=device)
|
||||
cu_seqlens = cu_seqlens.to(device=device, non_blocking=True)
|
||||
|
||||
max_seqlen = None
|
||||
if self.attn_backend in {
|
||||
@@ -1131,10 +1134,13 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
|
||||
siglip_position_ids.append(image_position_ids)
|
||||
cu_seqlens.append(cu_seqlens[-1] + numel)
|
||||
|
||||
# Both are built on the host; stage them over non-blocking.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values.device
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values.device
|
||||
)
|
||||
cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to(pixel_values.device)
|
||||
|
||||
vision_outputs = self.visual(
|
||||
pixel_values=pixel_values,
|
||||
|
||||
Reference in New Issue
Block a user