diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index e64d9e551b..4fa03261c4 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -248,13 +248,13 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]): feature_attention_mask=MultiModalFieldConfig.flat_from_sizes( "audio", chunk_counts, dim=0 ), - chunk_counts=MultiModalFieldConfig.batched("audio"), + chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True), ) return dict( audio_embeds=MultiModalFieldConfig.batched("audio"), input_features=MultiModalFieldConfig.batched("audio"), feature_attention_mask=MultiModalFieldConfig.batched("audio"), - chunk_counts=MultiModalFieldConfig.batched("audio"), + chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True), ) diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 1d457dd395..259d20f99a 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -48,6 +48,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.platforms import current_platform +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.outputs import LogprobsTensors from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p from vllm.v1.worker.gpu.attn_utils import build_attn_metadata @@ -728,10 +729,13 @@ class DiffusionGemmaRequestStates: max_num_reqs, canvas_length, hidden_size, dtype=torch.float32, device=device ) - def init_canvas(self, slot_indices_np: np.ndarray) -> None: - """Initialize canvas with random tokens for the given slots.""" - n = slot_indices_np.shape[0] - self.canvas[slot_indices_np] = torch.randint( + def init_canvas(self, slot_indices: torch.Tensor) -> None: + """Initialize canvas with random tokens for the given slots. + + `slot_indices` must already be on device to avoid a cpu->gpu sync. + """ + n = slot_indices.shape[0] + self.canvas[slot_indices] = torch.randint( 0, self.vocab_size, (n, self.canvas_length), @@ -740,15 +744,15 @@ class DiffusionGemmaRequestStates: ) def add_request(self, slot_idx: int) -> None: - self.is_encoder_phase[slot_idx] = True - self.init_canvas(torch.tensor([slot_idx], device=self.device)) - self.step[slot_idx] = 0 - self.accepted_canvas_history_len[slot_idx] = 0 + self.is_encoder_phase[slot_idx].fill_(True) + self.init_canvas(async_tensor_h2d([slot_idx], device=self.device)) + self.step[slot_idx].fill_(0) + self.accepted_canvas_history_len[slot_idx].fill_(0) self.self_conditioning_embeds[slot_idx] = 0 def remove_request(self, slot_idx: int) -> None: - self.is_encoder_phase[slot_idx] = False - self.accepted_canvas_history_len[slot_idx] = 0 + self.is_encoder_phase[slot_idx].fill_(False) + self.accepted_canvas_history_len[slot_idx].fill_(0) self.self_conditioning_embeds[slot_idx] = 0 @@ -1148,11 +1152,15 @@ class DiffusionSampler: ps = input_batch.idx_mapping_np[prefill_indices_np[done_prefill_np]] if len(ps) == 0: return - states.init_canvas(ps) - self.req_states.draft_tokens[ps, : self.canvas_length] = states.canvas[ps] + # Move the slot indices across once, up front: indexing a device + # tensor with a numpy array copies them over synchronously each time. ps_gpu = async_copy_to_gpu( ps.astype(np.int64), device=states.is_encoder_phase.device ) + states.init_canvas(ps_gpu) + self.req_states.draft_tokens[ps_gpu, : self.canvas_length] = states.canvas[ + ps_gpu + ] states.is_encoder_phase.index_fill_(0, ps_gpu, False) def _handle_prefill( diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 78c1512544..1ef236907f 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -69,6 +69,7 @@ from vllm.multimodal.processing.processor import ( ) from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape +from vllm.utils.torch_utils import async_tensor_h2d from .interfaces import ( MultiModalEmbeddings, @@ -2039,7 +2040,20 @@ class Gemma4ForConditionalGeneration( ) -> torch.Tensor | None: logits = self.language_model.compute_logits(hidden_states) if logits is not None and self._suppress_token_ids: - logits[:, self._suppress_token_ids] = -float("inf") + # Cache a per-device index tensor for the (static) suppressed-token + # set and use `index_fill_`, so neither the Python-list indices nor + # the scalar fill value take a host roundtrip per call. + cache = getattr(self, "_suppress_token_ids_cache", None) + if cache is None: + cache = {} + self._suppress_token_ids_cache = cache + suppress_idx = cache.get(logits.device) + if suppress_idx is None: + suppress_idx = async_tensor_h2d( + self._suppress_token_ids, dtype=torch.long, device=logits.device + ) + cache[logits.device] = suppress_idx + logits.index_fill_(1, suppress_idx, -float("inf")) return logits # ------------------------------------------------------------------ # diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 456a38dc36..0e1f079dfa 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -548,9 +548,9 @@ class Glm4vVisionEmbeddings(nn.Module): else: # Convert inputs to tensors if needed if isinstance(lengths, list): - lengths = torch.tensor(lengths, device=device, dtype=torch.long) + lengths = async_tensor_h2d(lengths, device=device, dtype=torch.long) if not isinstance(image_shapes, torch.Tensor): - image_shapes = torch.tensor( + image_shapes = async_tensor_h2d( image_shapes, device=device, dtype=torch.long ) diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index 8b3828a912..a447f3e2ee 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -426,12 +426,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo ), image_num_patches=MultiModalFieldConfig.batched("image"), image_embeds=MultiModalFieldConfig.batched("image"), - image_token_id=MultiModalFieldConfig.shared("image", num_images), + image_token_id=MultiModalFieldConfig.shared( + "image", num_images, keep_on_cpu=True + ), pixel_values_videos=MultiModalFieldConfig.flat_from_sizes( "video", video_num_patches ), video_num_patches=MultiModalFieldConfig.batched("video"), - video_token_id=MultiModalFieldConfig.shared("video", num_videos), + video_token_id=MultiModalFieldConfig.shared( + "video", num_videos, keep_on_cpu=True + ), ) def _get_prompt_updates( diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index eae9e66fb7..cf60275040 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -478,7 +478,9 @@ class InternVLMultiModalProcessor( "video", video_num_patches ), video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True), - video_token_id=MultiModalFieldConfig.shared("video", num_videos), + video_token_id=MultiModalFieldConfig.shared( + "video", num_videos, keep_on_cpu=True + ), ) def _get_mm_fields_config( diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index fd8cf4cbda..005af154c0 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -871,12 +871,12 @@ def _keye_field_config( return dict( pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes), image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes), - image_grid_thw=MultiModalFieldConfig.batched("image"), + image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True), pixel_values_videos=MultiModalFieldConfig.flat_from_sizes( "video", video_grid_sizes ), video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes), - video_grid_thw=MultiModalFieldConfig.batched("video"), + video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True), ) @@ -1291,8 +1291,9 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): image_grid_thw = image_input["image_grid_thw"] assert image_grid_thw.ndim == 2 - for idx, thaw in enumerate(image_grid_thw): - thw_tuple = tuple(thaw.detach().cpu().numpy().tolist()) + image_grid_thw_list = image_grid_thw.tolist() + for idx, thw in enumerate(image_grid_thw_list): + thw_tuple = tuple(thw) numel = np.prod(thw_tuple) image_grid_hws.append(thw_tuple) image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:]) diff --git a/vllm/model_executor/models/phi3v.py b/vllm/model_executor/models/phi3v.py index 95689ef321..1162c676ea 100644 --- a/vllm/model_executor/models/phi3v.py +++ b/vllm/model_executor/models/phi3v.py @@ -266,9 +266,10 @@ class Phi3HDImageEmbedding(nn.Module): ) batch_image_features_proj = [] + image_sizes_list = image_sizes.tolist() # need a for loop to process each image because of different image sizes # (patch arrangement is different for each image) - for i, img_size in enumerate(image_sizes): + for i, img_size in enumerate(image_sizes_list): h, w = img_size h_crop = h // 336 w_crop = w // 336 @@ -426,7 +427,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]): ) -> Mapping[str, MultiModalFieldConfig]: return dict( pixel_values=MultiModalFieldConfig.batched("image"), - image_sizes=MultiModalFieldConfig.batched("image"), + image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True), image_embeds=MultiModalFieldConfig.batched("image"), ) diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index 8bb2d3dea9..8ea7b58eaf 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -270,8 +270,7 @@ class Phi4MMImageEncoder(nn.Module): pixel_values = pixel_values.flatten(0, 1) img_features = self.get_img_features( - pixel_values, - image_attention_mask.type(torch.BoolTensor).flatten(0, 1).to(target_device), + pixel_values, image_attention_mask.bool().flatten(0, 1) ) base_feat_height_target = self.base_feat_height_target diff --git a/vllm/model_executor/models/step3_vl.py b/vllm/model_executor/models/step3_vl.py index 943549c46e..f9b22705ed 100644 --- a/vllm/model_executor/models/step3_vl.py +++ b/vllm/model_executor/models/step3_vl.py @@ -200,7 +200,7 @@ class Step3VLMultiModalProcessor(BaseMultiModalProcessor[Step3VLProcessingInfo]) patch_pixel_values=MultiModalFieldConfig.flat_from_sizes( "image", num_patches ), - num_patches=MultiModalFieldConfig.batched("image"), + num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True), patch_newline_mask=MultiModalFieldConfig.flat_from_sizes( "image", num_patches ), @@ -682,7 +682,8 @@ class Step3VLForConditionalGeneration( merged_image_features = [] cur_patch_idx = 0 - for i, num_patch in enumerate(num_patches): + num_patches_list = num_patches.tolist() + for i, num_patch in enumerate(num_patches_list): cur_feature = [] if num_patch > 0: patch_slice = patch_image_features[ diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 32e932e3b8..b860474035 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -442,6 +442,7 @@ class CommonAttentionMetadata: # Needed by FastPrefillAttentionBuilder logits_indices_padded: torch.Tensor | None = None num_logits_indices: int | None = None + max_logits_per_req: int | None = None # Needed by CrossAttentionBuilder encoder_seq_lens: torch.Tensor | None = None @@ -590,6 +591,7 @@ class CommonAttentionMetadata: else self.causal, logits_indices_padded=self.logits_indices_padded, num_logits_indices=self.num_logits_indices, + max_logits_per_req=self.max_logits_per_req, encoder_seq_lens=maybe_slice_reqs(self.encoder_seq_lens), encoder_seq_lens_cpu=maybe_slice_reqs(self.encoder_seq_lens_cpu), dcp_local_seq_lens=maybe_slice_reqs(self.dcp_local_seq_lens), diff --git a/vllm/v1/attention/backends/mamba2_attn.py b/vllm/v1/attention/backends/mamba2_attn.py index 30b641bd0d..f44de4131c 100644 --- a/vllm/v1/attention/backends/mamba2_attn.py +++ b/vllm/v1/attention/backends/mamba2_attn.py @@ -150,11 +150,15 @@ class Mamba2AttentionMetadataBuilder( # Compute seq_idx for prefill only if common.num_prefills > 0: - prep_initial_states = ( - torch.any(common.has_initial_states_p).item() - if common.has_initial_states_p is not None - else False - ) + prep_initial_states = False + if common.has_initial_states_p is not None: + # Same condition as `has_initial_states_p`, but derived from CPU + # data so it needs no D2H. `seq_lens_cpu_upper_bound` is precise + # for prefill rows, which is all this slice covers. + num_computed_tokens_p_cpu, _ = self._prefill_cpu_metadata( + common, common_attn_metadata + ) + prep_initial_states = bool((num_computed_tokens_p_cpu > 0).any()) cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p = ( self._build_chunk_metadata_tensors( diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index 2a607edd9e..6b7ac71ae8 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -324,6 +324,33 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): return cu_chunk_seqlen, seq_idx, last_chunk_indices + def _prefill_cpu_metadata( + self, + common: M, + common_attn_metadata: CommonAttentionMetadata, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Prefill context lengths and query offsets, from CPU data only. + + `seq_lens_cpu_upper_bound` is precise for prefill rows in all modes + (including async spec decode), so this avoids the D2H sync that + `compute_num_computed_tokens().cpu()` would force. + + Returns (num_computed_tokens_p_cpu, query_start_loc_p_cpu). + """ + seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound + assert seq_lens_cpu is not None + num_reqs = common.num_reqs + num_prefills = common.num_prefills + query_start_loc_p_cpu = ( + common_attn_metadata.query_start_loc_cpu[-num_prefills - 1 :] + - common.num_decode_tokens + ) + prefill_query_lens_cpu = query_start_loc_p_cpu[1:] - query_start_loc_p_cpu[:-1] + num_computed_tokens_p_cpu = ( + seq_lens_cpu[num_reqs - num_prefills : num_reqs] - prefill_query_lens_cpu + ) + return num_computed_tokens_p_cpu, query_start_loc_p_cpu + def _build_chunk_metadata_tensors( self, chunk_size: int, @@ -334,23 +361,10 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): Compute chunk metadata and return as device tensors. Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p). """ - num_reqs = common.num_reqs num_prefills = common.num_prefills - num_decode_tokens = common.num_decode_tokens - # Derive prefill context lengths from CPU data only. - # `seq_lens_cpu_upper_bound` is precise for prefill rows in all modes - # (including async spec decode), so this avoids the D2H sync that - # `compute_num_computed_tokens().cpu()` would force. - seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound - assert seq_lens_cpu is not None - query_start_loc_p_cpu = ( - common_attn_metadata.query_start_loc_cpu[-num_prefills - 1 :] - - num_decode_tokens - ) - prefill_query_lens_cpu = query_start_loc_p_cpu[1:] - query_start_loc_p_cpu[:-1] - num_computed_tokens_p_cpu = ( - seq_lens_cpu[num_reqs - num_prefills : num_reqs] - prefill_query_lens_cpu + num_computed_tokens_p_cpu, query_start_loc_p_cpu = self._prefill_cpu_metadata( + common, common_attn_metadata ) cu_chunk_seqlen, seq_idx, last_chunk_indices = self._compute_chunk_metadata( diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index 4b8276dcce..28364b57ab 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -539,8 +539,14 @@ def make_kv_sharing_fast_prefill_common_attn_metadata( decode_query_start_loc[:1].fill_(0) # Avoid sync from scalar assignment. decode_query_start_loc[1:] = torch.cumsum(num_decode_tokens, dim=0) - decode_max_query_len = int(num_decode_tokens.max().item()) - total_num_decode_tokens = int(num_decode_tokens.sum().item()) + + # `num_decode_tokens` is a histogram over `logits_indices`, so its total is + # just how many there were -- already known as a Python int. + total_num_decode_tokens = num_logits_indices + + # Largest per-request logits count. + decode_max_query_len = common_attn_metadata.max_logits_per_req + assert decode_max_query_len is not None common_attn_metadata = CommonAttentionMetadata( query_start_loc=decode_query_start_loc, diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 2f9c4f811e..8b16ae0b84 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2009,10 +2009,11 @@ class GPUModelRunner( ) -> tuple[ torch.Tensor, SpecDecodeMetadata | None, + int, ]: """ Returns: - tuple[logits_indices, spec_decode_metadata] + tuple[logits_indices, spec_decode_metadata, max_num_sampled_tokens] """ total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens assert total_num_scheduled_tokens > 0 @@ -2324,6 +2325,7 @@ class GPUModelRunner( return ( logits_indices, spec_decode_metadata, + int(num_sampled_tokens.max()), ) def _build_attention_metadata( @@ -2335,6 +2337,7 @@ class GPUModelRunner( num_reqs_padded: int | None = None, ubatch_slices: UBatchSlices | None = None, logits_indices: torch.Tensor | None = None, + max_num_sampled_tokens: int | None = None, use_spec_decode: bool = False, for_cudagraph_capture: bool = False, num_scheduled_tokens: dict[str, int] | None = None, @@ -2509,6 +2512,7 @@ class GPUModelRunner( if logits_indices is not None and self.cache_config.kv_sharing_fast_prefill: cm_base.num_logits_indices = logits_indices.size(0) + cm_base.max_logits_per_req = max_num_sampled_tokens cm_base.logits_indices_padded = self._prepare_kv_sharing_fast_prefill( logits_indices ) @@ -4341,9 +4345,8 @@ class GPUModelRunner( max_num_scheduled_tokens = int(num_scheduled_tokens_np.max()) num_tokens_unpadded = scheduler_output.total_num_scheduled_tokens - logits_indices, spec_decode_metadata = self._prepare_inputs( - scheduler_output, - num_scheduled_tokens_np, + logits_indices, spec_decode_metadata, max_num_sampled_tokens = ( + self._prepare_inputs(scheduler_output, num_scheduled_tokens_np) ) cascade_attn_prefix_lens = None @@ -4481,6 +4484,7 @@ class GPUModelRunner( max_query_len=max_num_scheduled_tokens, ubatch_slices=ubatch_slices_attn, logits_indices=logits_indices, + max_num_sampled_tokens=max_num_sampled_tokens, use_spec_decode=use_spec_decode, num_scheduled_tokens=scheduler_output.num_scheduled_tokens, cascade_attn_prefix_lens=cascade_attn_prefix_lens,