[Perf] Avoid more GPU<->CPU syncs in multimodal encoders (#52369)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -867,7 +867,7 @@ class DiffusionGemmaModelState(ModelState):
|
||||
self.diffusion_states.add_request(req_index)
|
||||
if not new_req_data.req_id.startswith("_warmup_"):
|
||||
prompt_len = len(new_req_data.prompt_token_ids)
|
||||
self.diffusion_states.prompt_len[req_index] = prompt_len
|
||||
self.diffusion_states.prompt_len[req_index].fill_(prompt_len)
|
||||
|
||||
def remove_request(self, req_id: str) -> None:
|
||||
idx = self._req_id_to_index.pop(req_id, None)
|
||||
@@ -1276,7 +1276,10 @@ class DiffusionSampler:
|
||||
# before canvas padding so phantom positions stay uniform.
|
||||
if num_decode > 0:
|
||||
top_k, top_p = self.sampling_states.get_top_k_top_p(
|
||||
decode_slots.repeat_interleave(valid_canvas_len), decode_slots_np
|
||||
decode_slots.repeat_interleave(
|
||||
valid_canvas_len, output_size=int(valid_canvas_len_np.sum())
|
||||
),
|
||||
decode_slots_np,
|
||||
)
|
||||
if top_k is not None or top_p is not None:
|
||||
logits = apply_top_k_top_p(logits.float(), top_k, top_p)
|
||||
|
||||
@@ -72,6 +72,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .ernie45_vl_moe import Ernie4_5_VLMoeForCausalLM
|
||||
@@ -446,6 +447,9 @@ class Ernie4_5_VisionTransformer(nn.Module):
|
||||
pos_ids = torch.cat(pos_ids, dim=0)
|
||||
max_grid_size = grid_thw[:, 1:].max()
|
||||
rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
|
||||
# `pos_ids` is built on the host; stage it over non-blocking so the
|
||||
# gather below doesn't index a device tensor with a CPU one.
|
||||
pos_ids = pos_ids.to(rotary_pos_emb_full.device, non_blocking=True)
|
||||
rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
|
||||
return rotary_pos_emb
|
||||
|
||||
@@ -530,7 +534,7 @@ class Ernie4_5_VisionTransformer(nn.Module):
|
||||
else:
|
||||
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
|
||||
|
||||
cu_seqlens = cu_seqlens.to(device)
|
||||
cu_seqlens = cu_seqlens.to(device, non_blocking=True)
|
||||
|
||||
return {
|
||||
"rotary_pos_emb": rotary_pos_emb,
|
||||
@@ -806,8 +810,8 @@ class VariableResolutionResamplerModel(nn.Module):
|
||||
b_offset + (temp_offset + 1) * spatial_size,
|
||||
)
|
||||
)
|
||||
slice_offsets = torch.tensor(np.concatenate(slice_offsets, axis=-1)).to(
|
||||
x.device
|
||||
slice_offsets = async_tensor_h2d(
|
||||
np.concatenate(slice_offsets, axis=-1), device=x.device
|
||||
)
|
||||
|
||||
slice_offsets2 = []
|
||||
@@ -823,8 +827,8 @@ class VariableResolutionResamplerModel(nn.Module):
|
||||
b_offset + (temp_offset + 1) * spatial_size,
|
||||
)
|
||||
)
|
||||
slice_offsets2 = torch.tensor(np.concatenate(slice_offsets2, axis=-1)).to(
|
||||
x.device
|
||||
slice_offsets2 = async_tensor_h2d(
|
||||
np.concatenate(slice_offsets2, axis=-1), device=x.device
|
||||
)
|
||||
|
||||
x_timestep_1 = torch.index_select(x, dim=0, index=slice_offsets)
|
||||
|
||||
@@ -546,14 +546,6 @@ class Glm4vVisionEmbeddings(nn.Module):
|
||||
0, hidden_size, device=device, dtype=pos_embed_weight.dtype
|
||||
)
|
||||
else:
|
||||
# Convert inputs to tensors if needed
|
||||
if isinstance(lengths, list):
|
||||
lengths = async_tensor_h2d(lengths, device=device, dtype=torch.long)
|
||||
if not isinstance(image_shapes, torch.Tensor):
|
||||
image_shapes = async_tensor_h2d(
|
||||
image_shapes, device=device, dtype=torch.long
|
||||
)
|
||||
|
||||
# Prepare 2D position embedding
|
||||
orig_size_sq = pos_embed_weight.shape[0]
|
||||
orig_size = int(orig_size_sq**0.5)
|
||||
@@ -564,32 +556,23 @@ class Glm4vVisionEmbeddings(nn.Module):
|
||||
.to(device=device, dtype=torch.float32)
|
||||
)
|
||||
|
||||
# Calculate target dimensions for each patch
|
||||
# Add bounds checking for data parallel mode
|
||||
if len(lengths) > image_shapes.shape[0]:
|
||||
# In data parallel mode, some GPUs might not have all
|
||||
# image shapes
|
||||
# Use available image shapes, cycling if necessary
|
||||
target_h_list = []
|
||||
target_w_list = []
|
||||
for i in range(len(lengths)):
|
||||
# Cycle through available shapes
|
||||
shape_idx = i % image_shapes.shape[0]
|
||||
target_h_list.append(image_shapes[shape_idx, 1].repeat(lengths[i]))
|
||||
target_w_list.append(image_shapes[shape_idx, 2].repeat(lengths[i]))
|
||||
target_h = torch.cat(target_h_list).to(
|
||||
device=device, dtype=torch.float32
|
||||
)
|
||||
target_w = torch.cat(target_w_list).to(
|
||||
device=device, dtype=torch.float32
|
||||
)
|
||||
else:
|
||||
target_h = torch.cat(
|
||||
[image_shapes[i, 1].repeat(lengths[i]) for i in range(len(lengths))]
|
||||
).to(device=device, dtype=torch.float32)
|
||||
target_w = torch.cat(
|
||||
[image_shapes[i, 2].repeat(lengths[i]) for i in range(len(lengths))]
|
||||
).to(device=device, dtype=torch.float32)
|
||||
# Calculate target dimensions for each patch. `lengths` and
|
||||
# `image_shapes` are host data, so expand them with numpy and move
|
||||
# the result across once rather than per-image.
|
||||
# Shapes are cycled: in data parallel mode some GPUs might not
|
||||
# have all image shapes.
|
||||
shapes_np = np.asarray(image_shapes)
|
||||
shape_idx = np.arange(len(lengths)) % shapes_np.shape[0]
|
||||
target_h = async_tensor_h2d(
|
||||
np.repeat(shapes_np[shape_idx, 1], lengths),
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
target_w = async_tensor_h2d(
|
||||
np.repeat(shapes_np[shape_idx, 2], lengths),
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
# Normalize coordinates to [-1, 1] range for grid_sample
|
||||
h_coords = h_coords.to(device=device, dtype=torch.float32)
|
||||
@@ -818,9 +801,7 @@ class Glm4vVisionTransformer(nn.Module):
|
||||
)
|
||||
|
||||
lengths = [h * w] * t
|
||||
image_shapes = async_tensor_h2d(
|
||||
[[t, h, w]], dtype=torch.long, device=device
|
||||
)
|
||||
image_shapes = [[t, h, w]]
|
||||
|
||||
# Build the coordinates on the host (cheap integer math) but move
|
||||
# them across pinned + non-blocking, so the consumer's
|
||||
|
||||
@@ -424,7 +424,7 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
pixel_values=MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", image_num_patches
|
||||
),
|
||||
image_num_patches=MultiModalFieldConfig.batched("image"),
|
||||
image_num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
image_embeds=MultiModalFieldConfig.batched("image"),
|
||||
image_token_id=MultiModalFieldConfig.shared(
|
||||
"image", num_images, keep_on_cpu=True
|
||||
@@ -432,7 +432,7 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
|
||||
"video", video_num_patches
|
||||
),
|
||||
video_num_patches=MultiModalFieldConfig.batched("video"),
|
||||
video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||
video_token_id=MultiModalFieldConfig.shared(
|
||||
"video", num_videos, keep_on_cpu=True
|
||||
),
|
||||
|
||||
@@ -59,6 +59,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -608,7 +609,9 @@ class KeyeSiglipEncoder(nn.Module):
|
||||
[height_position_ids, width_position_ids],
|
||||
dim=-1,
|
||||
)
|
||||
max_grid_size = pids.max() + 1
|
||||
# The ids are built from the grids above, so `h`/`w` bound them
|
||||
# and the table size is known on the host.
|
||||
max_grid_size = max(max(h, w) for _, h, w in flatten_image_grid_thw)
|
||||
rope_emb_max_grid = self.rotary_pos_emb(max_grid_size)
|
||||
rope_emb = rope_emb_max_grid[pids].flatten(1)
|
||||
rope_emb = rope_emb.repeat(1, 2)
|
||||
@@ -694,19 +697,19 @@ class KeyeSiglipVisionTransformer(nn.Module):
|
||||
|
||||
last_hidden_state = self.post_layernorm(last_hidden_state)
|
||||
|
||||
sample_hidden_state = list()
|
||||
if cu_seqlens is None:
|
||||
raise ValueError(
|
||||
"cu_seqlens cannot be None for "
|
||||
"SiglipVisionTransformer output processing."
|
||||
)
|
||||
for i in range(cu_seqlens.shape[0] - 1):
|
||||
start = cu_seqlens[i]
|
||||
end = cu_seqlens[i + 1]
|
||||
tensor = last_hidden_state[:, start:end, :].squeeze(0)
|
||||
sample_hidden_state.append(tensor)
|
||||
|
||||
return sample_hidden_state
|
||||
# `cu_seqlens` is the running sum of the per-image `t * h * w`, so the
|
||||
# split sizes are known on the host and slicing needs no device read.
|
||||
split_sizes = [
|
||||
int(np.prod(thw)) for thw in self.encoder.flatten_list(image_grid_thw)
|
||||
]
|
||||
return [
|
||||
tensor.squeeze(0) for tensor in last_hidden_state.split(split_sizes, dim=1)
|
||||
]
|
||||
|
||||
|
||||
class KeyeSiglipVisionModel(nn.Module):
|
||||
@@ -1307,13 +1310,17 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
||||
)
|
||||
else:
|
||||
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
|
||||
# These are all built on the host, so stage them over
|
||||
# non-blocking rather than forcing a sync per transfer.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values.device
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to(
|
||||
pixel_values.device
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values.device
|
||||
)
|
||||
sample_indices = torch.concat(sample_indices, dim=0).to(
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
sample_indices = torch.concat(sample_indices, dim=0).to(pixel_values.device)
|
||||
|
||||
image_embeds = self.visual(
|
||||
pixel_values=pixel_values,
|
||||
@@ -1357,14 +1364,15 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
||||
)
|
||||
else:
|
||||
pixel_values_videos = pixel_values_videos.type(self.visual.dtype)
|
||||
# These are host-built; stage them over non-blocking.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values_videos.device
|
||||
pixel_values_videos.device, non_blocking=True
|
||||
)
|
||||
cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to(
|
||||
pixel_values_videos.device
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values_videos.device
|
||||
)
|
||||
sample_indices = torch.concat(sample_indices, dim=0).to(
|
||||
pixel_values_videos.device
|
||||
pixel_values_videos.device, non_blocking=True
|
||||
)
|
||||
|
||||
video_embeds = self.visual(
|
||||
|
||||
@@ -68,6 +68,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .ernie45 import Ernie4_5ForCausalLM
|
||||
@@ -800,16 +801,18 @@ class SiglipEncoder(nn.Module):
|
||||
[height_position_ids, width_position_ids],
|
||||
dim=-1,
|
||||
)
|
||||
max_grid_size = pids.max() + 1
|
||||
# The ids are built from the grids above, so `h`/`w` bound them
|
||||
# and the table size is known on the host.
|
||||
max_grid_size = max(max(h, w) for _, h, w in flatten_image_grid_thw)
|
||||
rope_emb_max_grid = self.rotary_pos_emb(max_grid_size)
|
||||
rotary_pos_emb = rope_emb_max_grid[pids].flatten(1)
|
||||
|
||||
if cu_seqlens is None:
|
||||
raise ValueError("cu_seqlens cannot be None for SiglipEncoder.")
|
||||
if not isinstance(cu_seqlens, torch.Tensor):
|
||||
cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32, device=device)
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, dtype=torch.int32, device=device)
|
||||
else:
|
||||
cu_seqlens = cu_seqlens.to(device=device)
|
||||
cu_seqlens = cu_seqlens.to(device=device, non_blocking=True)
|
||||
|
||||
max_seqlen = None
|
||||
if self.attn_backend in {
|
||||
@@ -1131,10 +1134,13 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
|
||||
siglip_position_ids.append(image_position_ids)
|
||||
cu_seqlens.append(cu_seqlens[-1] + numel)
|
||||
|
||||
# Both are built on the host; stage them over non-blocking.
|
||||
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
|
||||
pixel_values.device
|
||||
pixel_values.device, non_blocking=True
|
||||
)
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values.device
|
||||
)
|
||||
cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to(pixel_values.device)
|
||||
|
||||
vision_outputs = self.visual(
|
||||
pixel_values=pixel_values,
|
||||
|
||||
@@ -905,7 +905,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
return dict(
|
||||
input_image_embeds=MultiModalFieldConfig.batched("image"),
|
||||
image_attention_mask=MultiModalFieldConfig.batched("image"),
|
||||
image_sizes=MultiModalFieldConfig.batched("image"),
|
||||
image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
num_img_tokens=MultiModalFieldConfig.batched("image"),
|
||||
input_audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
)
|
||||
|
||||
@@ -244,7 +244,11 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
# higher than the number of audio samples
|
||||
audio_features=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks),
|
||||
audio_token_len=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks),
|
||||
audio_lens=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks),
|
||||
# Only ever used to derive the encoder attention metadata on the
|
||||
# host, so keep it there.
|
||||
audio_lens=MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", num_chunks, keep_on_cpu=True
|
||||
),
|
||||
# num_chunks can convert audio_chunked to audio batch dimension
|
||||
audio_num_chunks=MultiModalFieldConfig.batched("audio"),
|
||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
|
||||
+5
-2
@@ -27,11 +27,14 @@ def prepare_chunk_indices(cu_seqlens: torch.Tensor, chunk_size: int) -> torch.Te
|
||||
for n in triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist()
|
||||
]
|
||||
)
|
||||
return torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1).to(cu_seqlens)
|
||||
chunk_indices = torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1)
|
||||
return chunk_indices.to(
|
||||
device=cu_seqlens.device, dtype=cu_seqlens.dtype, non_blocking=True
|
||||
)
|
||||
|
||||
|
||||
@tensor_cache
|
||||
def prepare_chunk_offsets(cu_seqlens: torch.Tensor, chunk_size: int) -> torch.Tensor:
|
||||
return torch.cat(
|
||||
[cu_seqlens.new_tensor([0]), triton.cdiv(prepare_lens(cu_seqlens), chunk_size)]
|
||||
[cu_seqlens.new_zeros(1), triton.cdiv(prepare_lens(cu_seqlens), chunk_size)]
|
||||
).cumsum(-1)
|
||||
|
||||
Reference in New Issue
Block a user