[LoRA] Add LoRA support for DeepSeek-V4 Flash Vision (#55897)

Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
linitra24
2026-09-13 07:10:54 -07:00
committed by GitHub
parent de500290ff
commit dd4c841070
3 changed files with 42 additions and 3 deletions
+1 -1
View File
@@ -547,7 +547,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ | | `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ | | `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ | | `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
| `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | | ✅︎ | | `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | ✅︎ | ✅︎ |
| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ | | `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
| `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ | | `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ |
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ | | `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
+1 -1
View File
@@ -179,7 +179,7 @@ def _lora_shrink(
assert weight.dtype in [torch.float16, torch.bfloat16] assert weight.dtype in [torch.float16, torch.bfloat16]
assert inputs.size(1) == lora_a_weights[0].size(-1) assert inputs.size(1) == lora_a_weights[0].size(-1)
assert inputs.is_contiguous() inputs = inputs.contiguous()
assert output_tensor.is_contiguous() assert output_tensor.is_contiguous()
# metadata sanity check # metadata sanity check
+40 -1
View File
@@ -25,9 +25,11 @@ from torch import nn
from vllm.model_executor.models.interfaces import ( from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings, MultiModalEmbeddings,
SupportsEagle3, SupportsEagle3,
SupportsLoRA,
SupportsMultiModal, SupportsMultiModal,
SupportsPP, SupportsPP,
) )
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.utils import ( from vllm.model_executor.models.utils import (
AutoWeightsLoader, AutoWeightsLoader,
WeightsMapper, WeightsMapper,
@@ -35,6 +37,7 @@ from vllm.model_executor.models.utils import (
maybe_prefix, maybe_prefix,
) )
from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItem
from .mm_preprocess import ( from .mm_preprocess import (
IMAGE_PLACEHOLDER, IMAGE_PLACEHOLDER,
@@ -81,7 +84,7 @@ def _make_deepseek_v4_vl_weights_mapper(
dummy_inputs=DeepseekV4VLDummyInputsBuilder, dummy_inputs=DeepseekV4VLDummyInputsBuilder,
) )
class DeepseekV4ForConditionalGeneration( class DeepseekV4ForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3 nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3, SupportsLoRA
): ):
"""Multimodal entry point for DeepSeek-V4 checkpoints with a vision tower. """Multimodal entry point for DeepSeek-V4 checkpoints with a vision tower.
@@ -89,9 +92,19 @@ class DeepseekV4ForConditionalGeneration(
delegates through ``language_model`` via the protocol defaults. delegates through ``language_model`` via the protocol defaults.
""" """
packed_modules_mapping = {
"gate_up_proj": ["w1", "w3"],
"fused_wqa_wkv": ["wq_a", "wkv"],
"fused_wkv_wgate": ["wkv", "wgate"],
# for visual encoder
"wqkv": ["wqkv"],
"w1": ["w1"],
}
# The MoE router needs raw token ids to detect image sentinel tokens # The MoE router needs raw token ids to detect image sentinel tokens
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl. # (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
requires_raw_input_tokens = True requires_raw_input_tokens = True
supports_tower_connector_lora = True
@classmethod @classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None: def get_placeholder_str(cls, modality: str, i: int) -> str | None:
@@ -329,3 +342,29 @@ class DeepseekV4ForConditionalGeneration(
return return
self.language_model.process_weights_after_loading() self.language_model.process_weights_after_loading()
self._weights_finalized = True self._weights_finalized = True
def get_mm_mapping(self) -> MultiModelKeys:
"""Get the module prefixes in the multimodal model."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="aligner",
tower_model="vision.",
)
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
if modality != "image":
raise ValueError(f"Unsupported modality: {modality!r}")
patches = mm_kwargs.get("patches") if mm_kwargs else None
if patches is not None and isinstance(patches.data, torch.Tensor):
tower_tokens = patches.data.shape[0]
else:
tower_tokens = num_mm_embeds * self.config.vision_downsample_ratio**2
return tower_tokens, num_mm_embeds