[MM][CG] Support ViT full CUDA graph for DeepSeek-VL2 (#46005)

Signed-off-by: littlecircle0730 <littlecircle0730@gmail.com>
Signed-off-by: littlecircle0730 <43994952+littlecircle0730@users.noreply.github.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
littlecircle0730
2026-10-15 21:38:52 +08:00
committed by GitHub
co-authored by Shanshan Shen
parent 0951fc183c
commit 71536bd5f8
5 changed files with 482 additions and 4 deletions
+193 -4
View File
@@ -4,9 +4,10 @@
# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py
"""Inference-only Deepseek-VL2 model compatible with HuggingFace weights."""
import functools
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, TypeAlias
from collections.abc import Hashable, Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, TypeAlias
import torch
import torch.nn as nn
@@ -51,7 +52,12 @@ from vllm.transformers_utils.configs.deepseek_vl2 import (
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import set_default_torch_dtype
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from .interfaces import (
MultiModalEmbeddings,
SupportsEncoderCudaGraph,
SupportsMultiModal,
SupportsPP,
)
from .utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -319,7 +325,9 @@ class DeepseekVL2MultiModalProcessor(
info=DeepseekVL2ProcessingInfo,
dummy_inputs=DeepseekVL2DummyInputsBuilder,
)
class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
class DeepseekVLV2ForCausalLM(
nn.Module, SupportsMultiModal, SupportsPP, SupportsEncoderCudaGraph
):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"language.": "language_model.",
@@ -340,6 +348,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
multimodal_config = vllm_config.model_config.multimodal_config
self.config = config
self.model_config = vllm_config.model_config
self.multimodal_config = multimodal_config
self.vision_config = config.vision_config
@@ -583,6 +592,186 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
pixel_values=pixel_values, images_spatial_crop=images_spatial_crop
)
# -- SupportsEncoderCudaGraph protocol methods --
@functools.cached_property
def _proj_hw(self) -> tuple[int, int]:
"""Return (h, w) of the projector output per tile."""
h = w = math.ceil(
(self.vision_config.image_size // self.vision_config.patch_size)
/ self.projector_config.downsample_ratio
)
return h, w
def get_encoder_cudagraph_config(self):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
return EncoderCudaGraphConfig(
modalities=["image"],
buffer_keys=["pixel_values"],
out_hidden_size=self.projector_config.n_embed,
)
def get_encoder_cudagraph_budget_range(
self, vllm_config: "VllmConfig"
) -> tuple[int, int]:
h, w = self._proj_hw
# Smallest valid image: 1 global + 1×1 local tile → 2 tiles total.
min_budget = h * (w + 1) * 2 + 1
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
self.model_config.max_model_len,
)
return min_budget, max_budget
def get_encoder_cudagraph_item_specs(self, mm_kwargs: dict[str, Any]):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
h, w = self._proj_hw
specs = []
for row in mm_kwargs["images_spatial_crop"].tolist():
tw, th = int(row[0]), int(row[1])
if tw == 0 or th == 0:
break
n_tiles = 1 + tw * th
output_tokens = h * (w + 1) + th * h * (tw * w + 1) + 1
specs.append(
EncoderItemSpec(input_size=n_tiles, output_tokens=output_tokens)
)
return specs
def select_encoder_cudagraph_items(
self, mm_kwargs: dict[str, Any], indices: list[int]
) -> dict[str, Any]:
pixel_values = mm_kwargs["pixel_values"]
images_spatial_crop = mm_kwargs["images_spatial_crop"]
if not indices:
return {
"pixel_values": pixel_values[:0],
"images_spatial_crop": images_spatial_crop[:0],
}
# Build per-image tile counts (1 global + tw*th local).
tiles_per_image = []
for row in images_spatial_crop.tolist():
tw, th = int(row[0]), int(row[1])
if tw == 0 or th == 0:
break
tiles_per_image.append(1 + tw * th)
cum_tiles = [0]
for t in tiles_per_image:
cum_tiles.append(cum_tiles[-1] + t)
selected_pv = torch.cat(
[pixel_values[cum_tiles[i] : cum_tiles[i + 1]] for i in indices]
)
return {
"pixel_values": selected_pv,
"images_spatial_crop": images_spatial_crop[list(indices)],
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
axis_keys: tuple[Hashable, ...] | None = None,
):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphCaptureInputs
h, w = self._proj_hw
# Each tile contributes at least h*w output tokens (before newlines).
# Using h*w (not h*(w+1)) guarantees we never under-allocate.
max_tiles = (token_budget + h * w - 1) // (h * w)
dummy_pixel_values = torch.randn(
max_tiles,
3,
self.vision_config.image_size,
self.vision_config.image_size,
device=device,
dtype=dtype,
)
return EncoderCudaGraphCaptureInputs(
values={"pixel_values": dummy_pixel_values}
)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
return EncoderCudaGraphReplayBuffers(
values={"pixel_values": mm_kwargs["pixel_values"]}
)
def encoder_cudagraph_forward(
self, values: dict[str, torch.Tensor], path: str = "default"
) -> torch.Tensor:
pixel_values = values["pixel_values"]
features = self.vision.forward_features(pixel_values)
return self.projector(features)
def postprocess_encoder_output(
self,
outputs: dict[str, torch.Tensor],
indices: list[int],
per_item_out_tokens: list[int],
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
clone: bool = False,
batch_mm_kwargs: dict[str, Any] | None = None,
) -> None:
"""Assemble per-image embeddings from raw per-tile projector output.
``outputs["default"]`` has shape ``[N_tiles_captured, hw, n_embed]``.
This method re-applies newlines and the view-separator that the
eager ``_pixel_values_to_embedding`` path inserts in-line.
"""
assert batch_mm_kwargs is not None
output = outputs["default"]
images_spatial_crop = batch_mm_kwargs["images_spatial_crop"]
_, hw, n_dim = output.shape
h = w = int(hw**0.5)
tile_offset = 0
for rank, img_idx in enumerate(indices):
tw, th = (
int(images_spatial_crop[rank][0]),
int(images_spatial_crop[rank][1]),
)
n_tiles = 1 + tw * th
tiles = output[tile_offset : tile_offset + n_tiles]
tile_offset += n_tiles
# Global tile: add one newline per row.
global_f = tiles[0].view(h, w, n_dim)
newline_g = self.image_newline.view(1, 1, n_dim).expand(h, 1, n_dim)
global_f = torch.cat([global_f, newline_g], dim=1).view(-1, n_dim)
# Local tiles: rearrange grid, then add one newline per row.
local_f = tiles[1:].view(th, tw, h, w, n_dim)
local_f = local_f.permute(0, 2, 1, 3, 4).reshape(th * h, tw * w, n_dim)
newline_l = self.image_newline.view(1, 1, n_dim).expand(th * h, 1, n_dim)
local_f = torch.cat([local_f, newline_l], dim=1).view(-1, n_dim)
if self.global_view_pos == "head":
emb = torch.cat([global_f, self.view_seperator[None], local_f])
else:
emb = torch.cat([local_f, self.view_seperator[None], global_f])
if isinstance(dest, dict):
dest[img_idx] = emb.clone() if clone else emb
else:
dest[rank] = emb.clone() if clone else emb
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None: