[MM][CG] Support ViT full CUDA graph for DeepSeek-VL2 (#46005)
Signed-off-by: littlecircle0730 <littlecircle0730@gmail.com> Signed-off-by: littlecircle0730 <43994952+littlecircle0730@users.noreply.github.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Shanshan Shen
parent
0951fc183c
commit
71536bd5f8
@@ -4,9 +4,10 @@
|
||||
# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py
|
||||
"""Inference-only Deepseek-VL2 model compatible with HuggingFace weights."""
|
||||
|
||||
import functools
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Literal, TypeAlias
|
||||
from collections.abc import Hashable, Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -51,7 +52,12 @@ from vllm.transformers_utils.configs.deepseek_vl2 import (
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
@@ -319,7 +325,9 @@ class DeepseekVL2MultiModalProcessor(
|
||||
info=DeepseekVL2ProcessingInfo,
|
||||
dummy_inputs=DeepseekVL2DummyInputsBuilder,
|
||||
)
|
||||
class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
class DeepseekVLV2ForCausalLM(
|
||||
nn.Module, SupportsMultiModal, SupportsPP, SupportsEncoderCudaGraph
|
||||
):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"language.": "language_model.",
|
||||
@@ -340,6 +348,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.multimodal_config = multimodal_config
|
||||
|
||||
self.vision_config = config.vision_config
|
||||
@@ -583,6 +592,186 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
pixel_values=pixel_values, images_spatial_crop=images_spatial_crop
|
||||
)
|
||||
|
||||
# -- SupportsEncoderCudaGraph protocol methods --
|
||||
|
||||
@functools.cached_property
|
||||
def _proj_hw(self) -> tuple[int, int]:
|
||||
"""Return (h, w) of the projector output per tile."""
|
||||
h = w = math.ceil(
|
||||
(self.vision_config.image_size // self.vision_config.patch_size)
|
||||
/ self.projector_config.downsample_ratio
|
||||
)
|
||||
return h, w
|
||||
|
||||
def get_encoder_cudagraph_config(self):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
|
||||
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
buffer_keys=["pixel_values"],
|
||||
out_hidden_size=self.projector_config.n_embed,
|
||||
)
|
||||
|
||||
def get_encoder_cudagraph_budget_range(
|
||||
self, vllm_config: "VllmConfig"
|
||||
) -> tuple[int, int]:
|
||||
h, w = self._proj_hw
|
||||
# Smallest valid image: 1 global + 1×1 local tile → 2 tiles total.
|
||||
min_budget = h * (w + 1) * 2 + 1
|
||||
max_budget = min(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
self.model_config.max_model_len,
|
||||
)
|
||||
return min_budget, max_budget
|
||||
|
||||
def get_encoder_cudagraph_item_specs(self, mm_kwargs: dict[str, Any]):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
|
||||
|
||||
h, w = self._proj_hw
|
||||
specs = []
|
||||
for row in mm_kwargs["images_spatial_crop"].tolist():
|
||||
tw, th = int(row[0]), int(row[1])
|
||||
if tw == 0 or th == 0:
|
||||
break
|
||||
n_tiles = 1 + tw * th
|
||||
output_tokens = h * (w + 1) + th * h * (tw * w + 1) + 1
|
||||
specs.append(
|
||||
EncoderItemSpec(input_size=n_tiles, output_tokens=output_tokens)
|
||||
)
|
||||
return specs
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self, mm_kwargs: dict[str, Any], indices: list[int]
|
||||
) -> dict[str, Any]:
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
images_spatial_crop = mm_kwargs["images_spatial_crop"]
|
||||
|
||||
if not indices:
|
||||
return {
|
||||
"pixel_values": pixel_values[:0],
|
||||
"images_spatial_crop": images_spatial_crop[:0],
|
||||
}
|
||||
|
||||
# Build per-image tile counts (1 global + tw*th local).
|
||||
tiles_per_image = []
|
||||
for row in images_spatial_crop.tolist():
|
||||
tw, th = int(row[0]), int(row[1])
|
||||
if tw == 0 or th == 0:
|
||||
break
|
||||
tiles_per_image.append(1 + tw * th)
|
||||
|
||||
cum_tiles = [0]
|
||||
for t in tiles_per_image:
|
||||
cum_tiles.append(cum_tiles[-1] + t)
|
||||
|
||||
selected_pv = torch.cat(
|
||||
[pixel_values[cum_tiles[i] : cum_tiles[i + 1]] for i in indices]
|
||||
)
|
||||
return {
|
||||
"pixel_values": selected_pv,
|
||||
"images_spatial_crop": images_spatial_crop[list(indices)],
|
||||
}
|
||||
|
||||
def prepare_encoder_cudagraph_capture_inputs(
|
||||
self,
|
||||
token_budget: int,
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
axis_keys: tuple[Hashable, ...] | None = None,
|
||||
):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphCaptureInputs
|
||||
|
||||
h, w = self._proj_hw
|
||||
# Each tile contributes at least h*w output tokens (before newlines).
|
||||
# Using h*w (not h*(w+1)) guarantees we never under-allocate.
|
||||
max_tiles = (token_budget + h * w - 1) // (h * w)
|
||||
dummy_pixel_values = torch.randn(
|
||||
max_tiles,
|
||||
3,
|
||||
self.vision_config.image_size,
|
||||
self.vision_config.image_size,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
return EncoderCudaGraphCaptureInputs(
|
||||
values={"pixel_values": dummy_pixel_values}
|
||||
)
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
path: str = "default",
|
||||
):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||
|
||||
return EncoderCudaGraphReplayBuffers(
|
||||
values={"pixel_values": mm_kwargs["pixel_values"]}
|
||||
)
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self, values: dict[str, torch.Tensor], path: str = "default"
|
||||
) -> torch.Tensor:
|
||||
pixel_values = values["pixel_values"]
|
||||
features = self.vision.forward_features(pixel_values)
|
||||
return self.projector(features)
|
||||
|
||||
def postprocess_encoder_output(
|
||||
self,
|
||||
outputs: dict[str, torch.Tensor],
|
||||
indices: list[int],
|
||||
per_item_out_tokens: list[int],
|
||||
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Assemble per-image embeddings from raw per-tile projector output.
|
||||
|
||||
``outputs["default"]`` has shape ``[N_tiles_captured, hw, n_embed]``.
|
||||
This method re-applies newlines and the view-separator that the
|
||||
eager ``_pixel_values_to_embedding`` path inserts in-line.
|
||||
"""
|
||||
assert batch_mm_kwargs is not None
|
||||
output = outputs["default"]
|
||||
images_spatial_crop = batch_mm_kwargs["images_spatial_crop"]
|
||||
_, hw, n_dim = output.shape
|
||||
h = w = int(hw**0.5)
|
||||
|
||||
tile_offset = 0
|
||||
for rank, img_idx in enumerate(indices):
|
||||
tw, th = (
|
||||
int(images_spatial_crop[rank][0]),
|
||||
int(images_spatial_crop[rank][1]),
|
||||
)
|
||||
n_tiles = 1 + tw * th
|
||||
tiles = output[tile_offset : tile_offset + n_tiles]
|
||||
tile_offset += n_tiles
|
||||
|
||||
# Global tile: add one newline per row.
|
||||
global_f = tiles[0].view(h, w, n_dim)
|
||||
newline_g = self.image_newline.view(1, 1, n_dim).expand(h, 1, n_dim)
|
||||
global_f = torch.cat([global_f, newline_g], dim=1).view(-1, n_dim)
|
||||
|
||||
# Local tiles: rearrange grid, then add one newline per row.
|
||||
local_f = tiles[1:].view(th, tw, h, w, n_dim)
|
||||
local_f = local_f.permute(0, 2, 1, 3, 4).reshape(th * h, tw * w, n_dim)
|
||||
newline_l = self.image_newline.view(1, 1, n_dim).expand(th * h, 1, n_dim)
|
||||
local_f = torch.cat([local_f, newline_l], dim=1).view(-1, n_dim)
|
||||
|
||||
if self.global_view_pos == "head":
|
||||
emb = torch.cat([global_f, self.view_seperator[None], local_f])
|
||||
else:
|
||||
emb = torch.cat([local_f, self.view_seperator[None], global_f])
|
||||
|
||||
if isinstance(dest, dict):
|
||||
dest[img_idx] = emb.clone() if clone else emb
|
||||
else:
|
||||
dest[rank] = emb.clone() if clone else emb
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is None:
|
||||
|
||||
Reference in New Issue
Block a user