diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 6395e4fddf..0fc32b99d8 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -867,7 +867,7 @@ class DiffusionGemmaModelState(ModelState): self.diffusion_states.add_request(req_index) if not new_req_data.req_id.startswith("_warmup_"): prompt_len = len(new_req_data.prompt_token_ids) - self.diffusion_states.prompt_len[req_index] = prompt_len + self.diffusion_states.prompt_len[req_index].fill_(prompt_len) def remove_request(self, req_id: str) -> None: idx = self._req_id_to_index.pop(req_id, None) @@ -1276,7 +1276,10 @@ class DiffusionSampler: # before canvas padding so phantom positions stay uniform. if num_decode > 0: top_k, top_p = self.sampling_states.get_top_k_top_p( - decode_slots.repeat_interleave(valid_canvas_len), decode_slots_np + decode_slots.repeat_interleave( + valid_canvas_len, output_size=int(valid_canvas_len_np.sum()) + ), + decode_slots_np, ) if top_k is not None or top_p is not None: logits = apply_top_k_top_p(logits.float(), top_k, top_p) diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index f27e047b47..f3ef38329b 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -72,6 +72,7 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backends.registry import AttentionBackendEnum from .ernie45_vl_moe import Ernie4_5_VLMoeForCausalLM @@ -446,6 +447,9 @@ class Ernie4_5_VisionTransformer(nn.Module): pos_ids = torch.cat(pos_ids, dim=0) max_grid_size = grid_thw[:, 1:].max() rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + # `pos_ids` is built on the host; stage it over non-blocking so the + # gather below doesn't index a device tensor with a CPU one. + pos_ids = pos_ids.to(rotary_pos_emb_full.device, non_blocking=True) rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb @@ -530,7 +534,7 @@ class Ernie4_5_VisionTransformer(nn.Module): else: max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens) - cu_seqlens = cu_seqlens.to(device) + cu_seqlens = cu_seqlens.to(device, non_blocking=True) return { "rotary_pos_emb": rotary_pos_emb, @@ -806,8 +810,8 @@ class VariableResolutionResamplerModel(nn.Module): b_offset + (temp_offset + 1) * spatial_size, ) ) - slice_offsets = torch.tensor(np.concatenate(slice_offsets, axis=-1)).to( - x.device + slice_offsets = async_tensor_h2d( + np.concatenate(slice_offsets, axis=-1), device=x.device ) slice_offsets2 = [] @@ -823,8 +827,8 @@ class VariableResolutionResamplerModel(nn.Module): b_offset + (temp_offset + 1) * spatial_size, ) ) - slice_offsets2 = torch.tensor(np.concatenate(slice_offsets2, axis=-1)).to( - x.device + slice_offsets2 = async_tensor_h2d( + np.concatenate(slice_offsets2, axis=-1), device=x.device ) x_timestep_1 = torch.index_select(x, dim=0, index=slice_offsets) diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 0e1f079dfa..679c90728b 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -546,14 +546,6 @@ class Glm4vVisionEmbeddings(nn.Module): 0, hidden_size, device=device, dtype=pos_embed_weight.dtype ) else: - # Convert inputs to tensors if needed - if isinstance(lengths, list): - lengths = async_tensor_h2d(lengths, device=device, dtype=torch.long) - if not isinstance(image_shapes, torch.Tensor): - image_shapes = async_tensor_h2d( - image_shapes, device=device, dtype=torch.long - ) - # Prepare 2D position embedding orig_size_sq = pos_embed_weight.shape[0] orig_size = int(orig_size_sq**0.5) @@ -564,32 +556,23 @@ class Glm4vVisionEmbeddings(nn.Module): .to(device=device, dtype=torch.float32) ) - # Calculate target dimensions for each patch - # Add bounds checking for data parallel mode - if len(lengths) > image_shapes.shape[0]: - # In data parallel mode, some GPUs might not have all - # image shapes - # Use available image shapes, cycling if necessary - target_h_list = [] - target_w_list = [] - for i in range(len(lengths)): - # Cycle through available shapes - shape_idx = i % image_shapes.shape[0] - target_h_list.append(image_shapes[shape_idx, 1].repeat(lengths[i])) - target_w_list.append(image_shapes[shape_idx, 2].repeat(lengths[i])) - target_h = torch.cat(target_h_list).to( - device=device, dtype=torch.float32 - ) - target_w = torch.cat(target_w_list).to( - device=device, dtype=torch.float32 - ) - else: - target_h = torch.cat( - [image_shapes[i, 1].repeat(lengths[i]) for i in range(len(lengths))] - ).to(device=device, dtype=torch.float32) - target_w = torch.cat( - [image_shapes[i, 2].repeat(lengths[i]) for i in range(len(lengths))] - ).to(device=device, dtype=torch.float32) + # Calculate target dimensions for each patch. `lengths` and + # `image_shapes` are host data, so expand them with numpy and move + # the result across once rather than per-image. + # Shapes are cycled: in data parallel mode some GPUs might not + # have all image shapes. + shapes_np = np.asarray(image_shapes) + shape_idx = np.arange(len(lengths)) % shapes_np.shape[0] + target_h = async_tensor_h2d( + np.repeat(shapes_np[shape_idx, 1], lengths), + device=device, + dtype=torch.float32, + ) + target_w = async_tensor_h2d( + np.repeat(shapes_np[shape_idx, 2], lengths), + device=device, + dtype=torch.float32, + ) # Normalize coordinates to [-1, 1] range for grid_sample h_coords = h_coords.to(device=device, dtype=torch.float32) @@ -818,9 +801,7 @@ class Glm4vVisionTransformer(nn.Module): ) lengths = [h * w] * t - image_shapes = async_tensor_h2d( - [[t, h, w]], dtype=torch.long, device=device - ) + image_shapes = [[t, h, w]] # Build the coordinates on the host (cheap integer math) but move # them across pinned + non-blocking, so the consumer's diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index a447f3e2ee..6dbc358e08 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -424,7 +424,7 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo pixel_values=MultiModalFieldConfig.flat_from_sizes( "image", image_num_patches ), - image_num_patches=MultiModalFieldConfig.batched("image"), + image_num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True), image_embeds=MultiModalFieldConfig.batched("image"), image_token_id=MultiModalFieldConfig.shared( "image", num_images, keep_on_cpu=True @@ -432,7 +432,7 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo pixel_values_videos=MultiModalFieldConfig.flat_from_sizes( "video", video_num_patches ), - video_num_patches=MultiModalFieldConfig.batched("video"), + video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True), video_token_id=MultiModalFieldConfig.shared( "video", num_videos, keep_on_cpu=True ), diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 005af154c0..28ccddd47e 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -59,6 +59,7 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape +from vllm.utils.torch_utils import async_tensor_h2d from .interfaces import ( MultiModalEmbeddings, @@ -608,7 +609,9 @@ class KeyeSiglipEncoder(nn.Module): [height_position_ids, width_position_ids], dim=-1, ) - max_grid_size = pids.max() + 1 + # The ids are built from the grids above, so `h`/`w` bound them + # and the table size is known on the host. + max_grid_size = max(max(h, w) for _, h, w in flatten_image_grid_thw) rope_emb_max_grid = self.rotary_pos_emb(max_grid_size) rope_emb = rope_emb_max_grid[pids].flatten(1) rope_emb = rope_emb.repeat(1, 2) @@ -694,19 +697,19 @@ class KeyeSiglipVisionTransformer(nn.Module): last_hidden_state = self.post_layernorm(last_hidden_state) - sample_hidden_state = list() if cu_seqlens is None: raise ValueError( "cu_seqlens cannot be None for " "SiglipVisionTransformer output processing." ) - for i in range(cu_seqlens.shape[0] - 1): - start = cu_seqlens[i] - end = cu_seqlens[i + 1] - tensor = last_hidden_state[:, start:end, :].squeeze(0) - sample_hidden_state.append(tensor) - - return sample_hidden_state + # `cu_seqlens` is the running sum of the per-image `t * h * w`, so the + # split sizes are known on the host and slicing needs no device read. + split_sizes = [ + int(np.prod(thw)) for thw in self.encoder.flatten_list(image_grid_thw) + ] + return [ + tensor.squeeze(0) for tensor in last_hidden_state.split(split_sizes, dim=1) + ] class KeyeSiglipVisionModel(nn.Module): @@ -1307,13 +1310,17 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): ) else: pixel_values = image_input["pixel_values"].type(self.visual.dtype) + # These are all built on the host, so stage them over + # non-blocking rather than forcing a sync per transfer. siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to( - pixel_values.device + pixel_values.device, non_blocking=True ) - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to( - pixel_values.device + cu_seqlens = async_tensor_h2d( + cu_seqlens, dtype=torch.int32, device=pixel_values.device + ) + sample_indices = torch.concat(sample_indices, dim=0).to( + pixel_values.device, non_blocking=True ) - sample_indices = torch.concat(sample_indices, dim=0).to(pixel_values.device) image_embeds = self.visual( pixel_values=pixel_values, @@ -1357,14 +1364,15 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): ) else: pixel_values_videos = pixel_values_videos.type(self.visual.dtype) + # These are host-built; stage them over non-blocking. siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to( - pixel_values_videos.device + pixel_values_videos.device, non_blocking=True ) - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to( - pixel_values_videos.device + cu_seqlens = async_tensor_h2d( + cu_seqlens, dtype=torch.int32, device=pixel_values_videos.device ) sample_indices = torch.concat(sample_indices, dim=0).to( - pixel_values_videos.device + pixel_values_videos.device, non_blocking=True ) video_embeds = self.visual( diff --git a/vllm/model_executor/models/paddleocr_vl.py b/vllm/model_executor/models/paddleocr_vl.py index 0dae22115d..bd635aaada 100644 --- a/vllm/model_executor/models/paddleocr_vl.py +++ b/vllm/model_executor/models/paddleocr_vl.py @@ -68,6 +68,7 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backends.registry import AttentionBackendEnum from .ernie45 import Ernie4_5ForCausalLM @@ -800,16 +801,18 @@ class SiglipEncoder(nn.Module): [height_position_ids, width_position_ids], dim=-1, ) - max_grid_size = pids.max() + 1 + # The ids are built from the grids above, so `h`/`w` bound them + # and the table size is known on the host. + max_grid_size = max(max(h, w) for _, h, w in flatten_image_grid_thw) rope_emb_max_grid = self.rotary_pos_emb(max_grid_size) rotary_pos_emb = rope_emb_max_grid[pids].flatten(1) if cu_seqlens is None: raise ValueError("cu_seqlens cannot be None for SiglipEncoder.") if not isinstance(cu_seqlens, torch.Tensor): - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32, device=device) + cu_seqlens = async_tensor_h2d(cu_seqlens, dtype=torch.int32, device=device) else: - cu_seqlens = cu_seqlens.to(device=device) + cu_seqlens = cu_seqlens.to(device=device, non_blocking=True) max_seqlen = None if self.attn_backend in { @@ -1131,10 +1134,13 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support siglip_position_ids.append(image_position_ids) cu_seqlens.append(cu_seqlens[-1] + numel) + # Both are built on the host; stage them over non-blocking. siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to( - pixel_values.device + pixel_values.device, non_blocking=True + ) + cu_seqlens = async_tensor_h2d( + cu_seqlens, dtype=torch.int32, device=pixel_values.device ) - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to(pixel_values.device) vision_outputs = self.visual( pixel_values=pixel_values, diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index 8ea7b58eaf..0fc10398a4 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -905,7 +905,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]): return dict( input_image_embeds=MultiModalFieldConfig.batched("image"), image_attention_mask=MultiModalFieldConfig.batched("image"), - image_sizes=MultiModalFieldConfig.batched("image"), + image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True), num_img_tokens=MultiModalFieldConfig.batched("image"), input_audio_embeds=MultiModalFieldConfig.batched("audio"), ) diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index ed3b6f1762..a2c051cf74 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -244,7 +244,11 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo # higher than the number of audio samples audio_features=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks), audio_token_len=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks), - audio_lens=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks), + # Only ever used to derive the encoder attention metadata on the + # host, so keep it there. + audio_lens=MultiModalFieldConfig.flat_from_sizes( + "audio", num_chunks, keep_on_cpu=True + ), # num_chunks can convert audio_chunked to audio batch dimension audio_num_chunks=MultiModalFieldConfig.batched("audio"), audio_embeds=MultiModalFieldConfig.batched("audio"), diff --git a/vllm/third_party/flash_linear_attention/ops/index.py b/vllm/third_party/flash_linear_attention/ops/index.py index 810d32c18b..8dc5f75f0f 100644 --- a/vllm/third_party/flash_linear_attention/ops/index.py +++ b/vllm/third_party/flash_linear_attention/ops/index.py @@ -27,11 +27,14 @@ def prepare_chunk_indices(cu_seqlens: torch.Tensor, chunk_size: int) -> torch.Te for n in triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist() ] ) - return torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1).to(cu_seqlens) + chunk_indices = torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1) + return chunk_indices.to( + device=cu_seqlens.device, dtype=cu_seqlens.dtype, non_blocking=True + ) @tensor_cache def prepare_chunk_offsets(cu_seqlens: torch.Tensor, chunk_size: int) -> torch.Tensor: return torch.cat( - [cu_seqlens.new_tensor([0]), triton.cdiv(prepare_lens(cu_seqlens), chunk_size)] + [cu_seqlens.new_zeros(1), triton.cdiv(prepare_lens(cu_seqlens), chunk_size)] ).cumsum(-1)