From 990f49bdcb8ff51c0ceb1d784c3ca16e6c276927 Mon Sep 17 00:00:00 2001 From: Soyaazz <523420504@qq.com> Date: Mon, 18 May 2026 11:19:13 +0800 Subject: [PATCH] [MM][CG] Enable encoder Cudagraph for Step3VL (#42224) Signed-off-by: JisoLya <523420504@qq.com> Signed-off-by: Soyaazz <523420504@qq.com> --- docs/design/cuda_graphs_multimodal.md | 2 + .../multimodal/vision_language_offline.py | 1 + .../generation/test_vit_cudagraph.py | 12 + vllm/model_executor/models/interfaces.py | 21 ++ vllm/model_executor/models/step3_vl.py | 325 +++++++++++++++++- vllm/model_executor/models/step_vl.py | 1 + vllm/model_executor/models/utils.py | 16 + vllm/v1/worker/encoder_cudagraph.py | 28 +- 8 files changed, 384 insertions(+), 22 deletions(-) diff --git a/docs/design/cuda_graphs_multimodal.md b/docs/design/cuda_graphs_multimodal.md index 2176f79a45..14781f6a5a 100644 --- a/docs/design/cuda_graphs_multimodal.md +++ b/docs/design/cuda_graphs_multimodal.md @@ -77,6 +77,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra * `encoder_eager_forward(...)` — fallback eager forward when no graph fits. * `get_input_modality(...)` - return the modality of the inputs. * `get_max_frames_per_video()` - return model-specific max frames per video. +* `postprocess_encoder_output(...)` - post process encoder output, directly call scatter_output_slices by default !!! note The `SupportsEncoderCudaGraph` protocol is designed to be model-agnostic. New vision encoder models can opt-in by implementing the protocol methods without modifying the manager. @@ -89,6 +90,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra | `Qwen2_5_VLForConditionalGeneration` | `Qwen2.5-VL` | ✅︎ | ✅︎ | | `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ | | `Qwen3_5ForConditionalGeneration` | `Qwen3.5` | ✅︎ | ✅︎ | +| `Step3VLForConditionalGeneration` | `Step3-VL` | ✅︎ | ❌︎ | !!! note Encoder CUDA Graphs have currently been tested with `--mm-encoder-attn-backend=FLASH_ATTN` and `--mm-encoder-attn-backend=FLASHINFER` on Blackwell GPUs. diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py index 1016259a1e..b4e34bd643 100644 --- a/examples/generate/multimodal/vision_language_offline.py +++ b/examples/generate/multimodal/vision_language_offline.py @@ -2560,6 +2560,7 @@ MODELS_SUPPORT_VIT_CUDA_GRAPH = [ "qwen2_vl", "qwen3_5", "qwen3_5_moe", + "stepvl", ] diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index dc29678207..342dcb8b6f 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -41,6 +41,13 @@ def qwen_vl_chat_template(content: str) -> str: return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n" +def step3_vl_chat_template(content: str) -> str: + return ( + "<|begin▁of▁sentence|> You are a helpful assistant.<|BOT|>user\n " + f"{content} <|EOT|><|BOT|>assistant\n" + ) + + MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { "qwen2_5_vl": VitCudagraphTestConfig( model="Qwen/Qwen2.5-VL-3B-Instruct", @@ -90,6 +97,11 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { needs_video_metadata=False, marks=[pytest.mark.core_model], ), + "step3_vl": VitCudagraphTestConfig( + model="stepfun-ai/Step3-VL-10B", + image_prompt=step3_vl_chat_template("What is in this image?"), + video_prompt=None, + ), } diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index d278af0607..000de94ee6 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -1594,6 +1594,27 @@ class SupportsEncoderCudaGraph(Protocol): """ ... + def postprocess_encoder_output( + self, + output: torch.Tensor, + indices: list[int], + per_item_out_tokens: list[int], + dest: dict[int, torch.Tensor] | list[torch.Tensor | None], + clone: bool = False, + batch_mm_kwargs: dict[str, Any] | None = None, + ) -> None: + """ + Post-process encoder output, directly call scatter_output_slices by default. + + By default, delegates directly to scatter_output_slices. + Override this for models that require additional processing on the raw + encoder output prior to scattering, e.g. Step3-VL, which merges features + according to dynamic patch counts before scattering. + """ + from vllm.model_executor.models.utils import scatter_output_slices + + scatter_output_slices(output, indices, per_item_out_tokens, dest, clone) + def prepare_encoder_cudagraph_capture_inputs( self, token_budget: int, diff --git a/vllm/model_executor/models/step3_vl.py b/vllm/model_executor/models/step3_vl.py index 8caa124552..36e0e73e36 100644 --- a/vllm/model_executor/models/step3_vl.py +++ b/vllm/model_executor/models/step3_vl.py @@ -46,7 +46,12 @@ from vllm.transformers_utils.processors.step3_vl import ( ) from vllm.utils.tensor_schema import TensorSchema, TensorShape -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP +from .interfaces import ( + MultiModalEmbeddings, + SupportsEncoderCudaGraph, + SupportsMultiModal, + SupportsPP, +) from .utils import ( AutoWeightsLoader, WeightsMapper, @@ -487,7 +492,9 @@ class Step3VisionTransformer(nn.Module): info=Step3VLProcessingInfo, dummy_inputs=Step3VLDummyInputsBuilder, ) -class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): +class Step3VLForConditionalGeneration( + nn.Module, SupportsMultiModal, SupportsPP, SupportsEncoderCudaGraph +): hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ "model.": "language_model.model.", @@ -510,6 +517,7 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP) multimodal_config = vllm_config.model_config.multimodal_config self.config = config + self.model_config = vllm_config.model_config self.multimodal_config = multimodal_config self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" @@ -570,6 +578,17 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP) def dtype(self): return next(self.parameters()).dtype + @staticmethod + def _compute_spatial_tokens(size, patch_size, stride): + # Compute the number of spatial tokens after two rounds of + # downsampling with given patch size and stride. + grid = size // patch_size + vit_tokens = grid * grid + spatial = int(math.sqrt(vit_tokens)) + h1 = (spatial - 2) // stride + 1 + h2 = (h1 - 1) // 2 + 1 + return h2 * h2 + def _parse_and_validate_image_input( self, **kwargs: object ) -> Step3VLImageInputs | None: @@ -676,6 +695,308 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP) is_multimodal=is_multimodal, ) + def get_encoder_cudagraph_config(self): + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphConfig, + ) + + return EncoderCudaGraphConfig( + modalities=["image"], + input_key_by_modality={"image": "pixel_values"}, + buffer_keys=["patch_pixel_values"], + out_hidden_size=self.config.hidden_size, + ) + + def get_input_modality( + self, + mm_kwargs: dict[str, Any], + ) -> str: + return "image" + + def get_max_frames_per_video( + self, + ) -> int: + return 0 + + def get_encoder_cudagraph_budget_range( + self, + vllm_config: "VllmConfig", + ) -> tuple[int, int]: + # An image without patches + min_budget = self._compute_spatial_tokens( + self.config.vision_config.image_size, + self.config.vision_config.patch_size, + self.config.understand_projector_stride, + ) + max_budget = min( + vllm_config.scheduler_config.max_num_batched_tokens, + self.model_config.max_model_len, + ) + return min_budget, max_budget + + def get_encoder_cudagraph_num_items( + self, + mm_kwargs: dict[str, Any], + ) -> int: + return len(mm_kwargs.get("pixel_values", [])) + + def get_encoder_cudagraph_per_item_output_tokens( + self, + mm_kwargs: dict[str, Any], + ) -> list[int]: + num_patches = mm_kwargs.get("num_patches") + img_output_tokens = self._compute_spatial_tokens( + self.config.vision_config.image_size, + self.config.vision_config.patch_size, + self.config.understand_projector_stride, + ) + patch_output_tokens = self._compute_spatial_tokens( + 504, + self.config.vision_config.patch_size, + self.config.understand_projector_stride, + ) + return [ + img_output_tokens + num_patch * patch_output_tokens + for num_patch in num_patches + ] + + def get_encoder_cudagraph_per_item_input_sizes( + self, + mm_kwargs: dict[str, Any], + ) -> list[int]: + img_grid = ( + self.config.vision_config.image_size // self.config.vision_config.patch_size + ) + + # NOTE: 504 is the hard coded size for each patch after processing + # by the vision model, which is determined by the current architecture + # of the vision model and may need to be updated if the architecture changes. + # The number of tokens for each patch is calculated based on this + # size and the patch size. + patch_grid = 504 // self.config.vision_config.patch_size + total_image_pixel = img_grid * img_grid + total_patch_pixel = patch_grid * patch_grid + + num_patches = mm_kwargs.get("num_patches") + return [ + total_image_pixel + num_patch * total_patch_pixel + for num_patch in num_patches + ] + + def select_encoder_cudagraph_items( + self, + mm_kwargs: dict[str, Any], + indices: list[int], + ) -> dict[str, Any]: + pixel_values = mm_kwargs["pixel_values"] + patch_pixel_values = mm_kwargs["patch_pixel_values"] + num_patches = mm_kwargs["num_patches"] + + # calcute the accumulated patch counts + cum_patches = [0] + for p in num_patches: + cum_patches.append(cum_patches[-1] + p) + + if len(indices) == 0: + return { + "pixel_values": pixel_values[:0], + "patch_pixel_values": patch_pixel_values[:0], + "num_patches": num_patches[:0], + } + + selected_pv = pixel_values[indices] + selected_np = num_patches[indices] + selected_ppv = torch.cat( + [patch_pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices] + ) + + return { + "pixel_values": selected_pv, + "patch_pixel_values": selected_ppv, + "num_patches": selected_np, + } + + def prepare_encoder_cudagraph_capture_inputs( + self, + token_budget: int, + max_batch_size: int, + max_frames_per_batch: int, + device: torch.device, + dtype: torch.dtype, + ): + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphCaptureInputs, + ) + + # For pixel_value, the max input size is max_batch_size + img_output_tokens = self._compute_spatial_tokens( + self.config.vision_config.image_size, + self.config.vision_config.patch_size, + self.config.understand_projector_stride, + ) + patch_output_tokens = self._compute_spatial_tokens( + 504, + self.config.vision_config.patch_size, + self.config.understand_projector_stride, + ) + dummy_pixel_values = torch.randn( + max_batch_size, + 3, + self.config.vision_config.image_size, + self.config.vision_config.image_size, + device=device, + dtype=dtype, + ) + # max_num_patches is the max total patches across the whole batch. + # token_budget = max_batch_size * img_out + max_num_patches * patch_out + max_num_patches = max( + 0, + (token_budget - max_batch_size * img_output_tokens) // patch_output_tokens, + ) + dummy_patch_pixel_values = torch.randn( + max_num_patches, + 3, + 504, + 504, + device=device, + dtype=dtype, + ) + # num_patches is NOT in buffers -- the per-item merge is done + # CPU-side by finalize_encoder_cudagraph_output using the actual + # batch's num_patches from mm_kwargs. + mm_kwargs = { + "pixel_values": dummy_pixel_values, + "patch_pixel_values": dummy_patch_pixel_values, + } + + buffers = { + "patch_pixel_values": dummy_patch_pixel_values, + } + + return EncoderCudaGraphCaptureInputs( + mm_kwargs=mm_kwargs, + buffers=buffers, + ) + + def encoder_cudagraph_forward( + self, + mm_kwargs: dict[str, Any], + buffers: dict[str, torch.Tensor], + ) -> torch.Tensor: + # Graph captures only the compute (vision model + conv projector). + # Per-item merge happens CPU-side in finalize_encoder_cudagraph_output + # using actual num_patches from the batch data. + pixel_values = mm_kwargs["pixel_values"] + patch_pixel_values = buffers["patch_pixel_values"] + + image_features = self._process_image_features( + self._get_vision_model_output(pixel_values) + ) + + has_patches = len(patch_pixel_values) > 0 + if has_patches: + patch_features = self._process_image_features( + self._get_vision_model_output(patch_pixel_values) + ) + + # Deterministic single cat: [all_img_flat, all_patch_flat] + img_flat = image_features.reshape(-1, image_features.shape[-1]) + if has_patches: + patch_flat = patch_features.reshape(-1, patch_features.shape[-1]) + return torch.cat([img_flat, patch_flat], dim=0) + return img_flat + + def encoder_eager_forward( + self, + mm_kwargs: dict[str, Any], + ) -> torch.Tensor: + image_input = Step3VLImagePixelInputs( + type="pixel_values", + pixel_values=mm_kwargs["pixel_values"], + patch_pixel_values=mm_kwargs["patch_pixel_values"], + num_patches=mm_kwargs["num_patches"], + ) + vision_embeddings = self._process_image_input(image_input) + return torch.cat(vision_embeddings, dim=0) + + def postprocess_encoder_output( + self, + output: torch.Tensor, + indices: list[int], + per_item_out_tokens: list[int], + dest: dict[int, torch.Tensor] | list[torch.Tensor | None], + clone: bool = False, + batch_mm_kwargs: dict[str, Any] | None = None, + ): + """CPU-side per-item merge after graph replay. + + The graph output is ``[all_img_flat, all_patch_flat]``. + This method splits the flat output into image and patch features, + then reassembles per-item embeddings using the *actual* batch + ``num_patches`` from ``batch_mm_kwargs`` (not the capture-time values). + """ + num_patches = batch_mm_kwargs["num_patches"] + hidden = output.shape[-1] + bsz = len(indices) + + img_out = self._compute_spatial_tokens( + self.config.vision_config.image_size, + self.config.vision_config.patch_size, + self.config.understand_projector_stride, + ) + patch_out = self._compute_spatial_tokens( + 504, + self.config.vision_config.patch_size, + self.config.understand_projector_stride, + ) + + # Valid portion: bsz images, actual_total_patches patches + actual_np = [int(np) for np in num_patches] + total_patches = sum(actual_np) + img_tokens = bsz * img_out + patch_tokens = total_patches * patch_out + + img_part = output[:img_tokens].reshape(bsz, img_out, hidden) + if total_patches > 0: + patch_part = output[img_tokens : img_tokens + patch_tokens].reshape( + -1, patch_out, hidden + ) + else: + patch_part = None + + merged: dict[int, torch.Tensor] = {} + cur_patch = 0 + for i, idx in enumerate(indices): + np = actual_np[i] + parts: list[torch.Tensor] = [] + if patch_part is not None and np > 0: + parts.append(patch_part[cur_patch : cur_patch + np].reshape(-1, hidden)) + cur_patch += np + parts.append(img_part[i].reshape(-1, hidden)) + merged[idx] = torch.cat(parts, dim=0) if len(parts) > 1 else parts[0] + + out = [merged[i] for i in indices] + for i, idx in enumerate(indices): + dest[idx] = out[i] + + def prepare_encoder_cudagraph_replay_buffers( + self, + mm_kwargs: dict[str, Any], + max_batch_size: int, + max_frames_per_batch: int, + ): + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphReplayBuffers, + ) + + # Only patch_pixel_values lives in the buffers dict; num_patches is + # processed CPU-side by finalize_encoder_cudagraph_output. + return EncoderCudaGraphReplayBuffers( + buffers={ + "patch_pixel_values": mm_kwargs["patch_pixel_values"], + }, + ) + def forward( self, input_ids: torch.Tensor | None, diff --git a/vllm/model_executor/models/step_vl.py b/vllm/model_executor/models/step_vl.py index 4669771f4b..53dd3ebf48 100644 --- a/vllm/model_executor/models/step_vl.py +++ b/vllm/model_executor/models/step_vl.py @@ -500,6 +500,7 @@ class StepVLForConditionalGeneration(Step3VLForConditionalGeneration): quant_config = vllm_config.quant_config self.config = config + self.model_config = vllm_config.model_config self.multimodal_config = multimodal_config self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index a6c4633930..48ea18ee19 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -884,3 +884,19 @@ def get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int: if feature_layer_index < 0: return num_hidden_layers + feature_layer_index + 1 return feature_layer_index + + +def scatter_output_slices( + output: torch.Tensor, + indices: list[int], + per_item_out_tokens: list[int], + dest: dict[int, torch.Tensor] | list[torch.Tensor | None], + clone: bool = False, +) -> None: + """Slice a concatenated output tensor and scatter into dest by index.""" + offset = 0 + for idx in indices: + n_tok = per_item_out_tokens[idx] + sliced = output[offset : offset + n_tok] + dest[idx] = sliced.clone() if clone else sliced + offset += n_tok diff --git a/vllm/v1/worker/encoder_cudagraph.py b/vllm/v1/worker/encoder_cudagraph.py index dd4d7262a7..4794fa7742 100644 --- a/vllm/v1/worker/encoder_cudagraph.py +++ b/vllm/v1/worker/encoder_cudagraph.py @@ -14,7 +14,10 @@ from vllm.distributed import ( tensor_model_parallel_all_gather, ) from vllm.logger import init_logger -from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph +from vllm.model_executor.models.interfaces import ( + SupportsEncoderCudaGraph, +) +from vllm.model_executor.models.utils import scatter_output_slices from vllm.model_executor.models.vision import get_load_balance_assignment from vllm.v1.worker.encoder_cudagraph_defs import ( EncoderCudaGraphConfig, @@ -254,22 +257,6 @@ class EncoderCudaGraphManager: for t in self.model.get_encoder_cudagraph_per_item_output_tokens(mm_kwargs) ] - @staticmethod - def _scatter_output_slices( - output: torch.Tensor, - indices: list[int], - per_item_out_tokens: list[int], - dest: dict[int, torch.Tensor] | list[torch.Tensor | None], - clone: bool = False, - ) -> None: - """Slice a concatenated output tensor and scatter into dest by index.""" - offset = 0 - for idx in indices: - n_tok = per_item_out_tokens[idx] - sliced = output[offset : offset + n_tok] - dest[idx] = sliced.clone() if clone else sliced - offset += n_tok - def _run_budget_graph( self, mm_kwargs: dict[str, Any], @@ -414,7 +401,7 @@ class EncoderCudaGraphManager: self.graph_misses += len(batch_orig_indices) with torch.inference_mode(): raw = self.model.encoder_eager_forward(batch_mm_kwargs) - self._scatter_output_slices( + scatter_output_slices( raw, batch_orig_indices, per_item_out_tokens, @@ -440,12 +427,13 @@ class EncoderCudaGraphManager: batch_mm_kwargs, token_budget, replay.buffers ) assert output is not None - self._scatter_output_slices( + self.model.postprocess_encoder_output( output, batch_orig_indices, per_item_out_tokens, outputs_by_orig_idx, clone=True, + batch_mm_kwargs=batch_mm_kwargs, ) # Return in original batch order (caller maps outputs to token positions) @@ -573,7 +561,7 @@ class EncoderCudaGraphManager: count = images_per_rank[rank] if count > 0: rank_items = image_rank_assignment[current_idx : current_idx + count] - self._scatter_output_slices( + scatter_output_slices( rank_outputs[rank], rank_items, per_item_out_tokens,