diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index bd8ec29bc3..76cae14aed 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -85,40 +85,42 @@ class TritonAttentionMetadata: scheduler_metadata: torch.Tensor | None = None prefix_scheduler_metadata: torch.Tensor | None = None mm_prefix_range: dict[int, list[tuple[int, int]]] | None = None + mm_prefix_range_tensor: torch.Tensor | None = None - @property - def mm_prefix_range_tensor(self) -> torch.Tensor | None: + @staticmethod + def compute_mm_prefix_range_tensor( + mm_prefix_range: dict[int, list[tuple[int, int]]] | None, + num_seqs: int, + device: torch.device, + ) -> torch.Tensor | None: """Convert mm_prefix_range dict to padded tensor for Triton kernel. Returns shape: (num_seqs, max_ranges, 2) with 0-padding for empty ranges. Empty ranges have start==end==0, which kernel skips via is_valid check. """ - # TODO(Isotr0py): Move to model runner's attention metadata - # preparation to avoid duplicate computation. - if self.mm_prefix_range is None: + if mm_prefix_range is None: return None - num_seqs = self.seq_lens.shape[0] - device = self.seq_lens.device - # Collect ranges, using [(0,0)] for empty sequences to ensure uniform dims range_lists = [ - self.mm_prefix_range.get(i, [(0, 0)]) or [(0, 0)] for i in range(num_seqs) + mm_prefix_range.get(i, [(0, 0)]) or [(0, 0)] for i in range(num_seqs) ] # Return None if all ranges are trivial (only (0,0) placeholders) if all(r == [(0, 0)] for r in range_lists): return None - # Create 2D tensors with shape (num_ranges, 2) for each sequence - range_tensors = [ - torch.tensor(r, dtype=torch.int32, device=device).view(-1, 2) - for r in range_lists - ] - - return torch.nested.nested_tensor( - range_tensors, layout=torch.jagged - ).to_padded_tensor(0) + # Build on CPU first then move to GPU in a single H2D transfer + max_ranges = max(len(r) for r in range_lists) + # Pad all sequences to the same number of ranges + padded = [] + for r in range_lists: + padded_r = list(r) + [(0, 0)] * (max_ranges - len(r)) + padded.append(padded_r) + # Create tensor with efficient H2D transfer + return torch.tensor(padded, dtype=torch.int32, device=device).view( + num_seqs, max_ranges, 2 + ) class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMetadata]): diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 872cb83d24..8dcac7bd08 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2332,13 +2332,8 @@ class GPUModelRunner( req_idx = self.input_batch.req_id_to_index[req_id] req_doc_ranges[req_idx] = image_doc_ranges - if isinstance(attn_metadata, list): - for ub_metadata in attn_metadata: - for _metadata in ub_metadata.values(): - _metadata.mm_prefix_range = req_doc_ranges # type: ignore[attr-defined] - else: - for _metadata in attn_metadata.values(): - _metadata.mm_prefix_range = req_doc_ranges # type: ignore[attr-defined] + # Set mm_prefix_range for all attention metadata + self._set_mm_prefix_range_for_metadata(attn_metadata, req_doc_ranges) if spec_decode_common_attn_metadata is not None and ( num_reqs != num_reqs_padded or num_tokens != num_tokens_padded @@ -6487,6 +6482,46 @@ class GPUModelRunner( return self.reorder_batch_threshold = reduce(min_none_high, reorder_batch_thresholds) # type: ignore[assignment] + def _set_mm_prefix_range_for_metadata( + self, + attn_metadata: Any, + req_doc_ranges: dict[int, list[tuple[int, int]]], + ) -> None: + """Set mm_prefix_range for all attention metadata objects. + + This method handles both list and non-list attention metadata, + computing mm_prefix_range_tensor once and sharing it across all + metadata objects to avoid redundant host-to-device transfers. + """ + from vllm.v1.attention.backends.triton_attn import ( + TritonAttentionMetadata, + ) + + # Get all metadata objects from either list or dict structure + metadata_list = [] + if isinstance(attn_metadata, list): + for ub_metadata in attn_metadata: + metadata_list.extend(ub_metadata.values()) + else: + metadata_list.extend(attn_metadata.values()) + + # Set mm_prefix_range for all metadata and compute tensor once + shared_tensor = None + for metadata in metadata_list: + metadata.mm_prefix_range = req_doc_ranges # type: ignore[attr-defined] + + # Only compute tensor for TritonAttentionMetadata + if isinstance(metadata, TritonAttentionMetadata): + if shared_tensor is None: + shared_tensor = ( + TritonAttentionMetadata.compute_mm_prefix_range_tensor( + req_doc_ranges, + metadata.seq_lens.shape[0], # type: ignore[attr-defined] + metadata.seq_lens.device, # type: ignore[attr-defined] + ) + ) + metadata.mm_prefix_range_tensor = shared_tensor + def may_reinitialize_input_batch( self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int] ) -> None: