[DSV4.1] Add encoder cuda graph support for deepseek-v4.1-flash (#56625)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-20 21:51:13 +08:00
committed by GitHub
parent 9b2f34cad4
commit 27757dde02
6 changed files with 599 additions and 63 deletions
+12 -28
View File
@@ -26,6 +26,7 @@ from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsEagle3,
SupportsEncoderCudaGraph,
SupportsMultiModal,
SupportsPP,
)
@@ -44,15 +45,12 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from ..common.mm_preprocess import (
IMAGE,
IMAGE_END,
IMAGE_NEW_LINE,
IMAGE_PLACEHOLDER,
IMAGE_START,
DeepseekV4VLDummyInputsBuilder,
DeepseekV4VLMultiModalProcessor,
DeepseekV4VLProcessingInfo,
)
from ..common.vl_cudagraph import DeepseekV4VLEncoderCudaGraphMixin
from .model import (
DeepseekV41LLMForCausalLM,
_linear_scale_param_name,
@@ -115,11 +113,20 @@ def _make_deepseek_v4_vl_weights_mapper(
info=DeepseekV4VLProcessingInfo,
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
)
class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3):
class DeepseekV41ForCausalLM(
nn.Module,
DeepseekV4VLEncoderCudaGraphMixin,
SupportsMultiModal,
SupportsEncoderCudaGraph,
SupportsPP,
SupportsEagle3,
):
"""Multimodal entry point for DeepSeek-V4.1 checkpoints with a vision tower.
``SupportsEagle3`` (aux hidden-state plumbing for MTP/DSpark drafters)
delegates through ``language_model`` via the protocol defaults.
``SupportsEncoderCudaGraph`` (ViT CUDA graph capture/replay) is
implemented by ``DeepseekV4VLEncoderCudaGraphMixin``.
"""
supports_encoder_tp_data = True
@@ -196,29 +203,6 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
resolve_bindings={"p": self.config.vision_patch_size},
)
def _encode_image(
self,
patches: torch.Tensor,
n_vit_h: int,
n_vit_w: int,
) -> torch.Tensor:
# Aligner rows in reading order, one per IMAGE slot.
return self.aligner(self.vision(patches, n_vit_h, n_vit_w), n_vit_h, n_vit_w)
def _build_image_span(
self, image_embeds: torch.Tensor, types: torch.Tensor
) -> torch.Tensor:
"""Full image span: aligner rows at IMAGE slots, the learned
delimiter vectors at IMAGE_START/IMAGE_NEW_LINE/IMAGE_END."""
types = types.to(image_embeds.device)
span = image_embeds.new_empty(types.numel(), image_embeds.shape[-1])
dtype = image_embeds.dtype
span[types == IMAGE_START] = self.image_start.to(dtype)
span[types == IMAGE_END] = self.image_end.to(dtype)
span[types == IMAGE_NEW_LINE] = self.image_newline.to(dtype)
span[types == IMAGE] = image_embeds
return span
def _process_image_input(
self,
image_input: DeepseekV4VLImagePixelInputs,