[LoRA] Add LoRA support for DeepSeek-V4 Flash Vision (#55897)
Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
@@ -547,7 +547,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
|||||||
| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
|
| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
|
||||||
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
|
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
|
||||||
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
|
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
|
||||||
| `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | | ✅︎ |
|
| `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | ✅︎ | ✅︎ |
|
||||||
| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
|
| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
|
||||||
| `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ |
|
| `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ |
|
||||||
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
|
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
|
||||||
|
|||||||
@@ -179,7 +179,7 @@ def _lora_shrink(
|
|||||||
assert weight.dtype in [torch.float16, torch.bfloat16]
|
assert weight.dtype in [torch.float16, torch.bfloat16]
|
||||||
|
|
||||||
assert inputs.size(1) == lora_a_weights[0].size(-1)
|
assert inputs.size(1) == lora_a_weights[0].size(-1)
|
||||||
assert inputs.is_contiguous()
|
inputs = inputs.contiguous()
|
||||||
assert output_tensor.is_contiguous()
|
assert output_tensor.is_contiguous()
|
||||||
|
|
||||||
# metadata sanity check
|
# metadata sanity check
|
||||||
|
|||||||
@@ -25,9 +25,11 @@ from torch import nn
|
|||||||
from vllm.model_executor.models.interfaces import (
|
from vllm.model_executor.models.interfaces import (
|
||||||
MultiModalEmbeddings,
|
MultiModalEmbeddings,
|
||||||
SupportsEagle3,
|
SupportsEagle3,
|
||||||
|
SupportsLoRA,
|
||||||
SupportsMultiModal,
|
SupportsMultiModal,
|
||||||
SupportsPP,
|
SupportsPP,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||||
from vllm.model_executor.models.utils import (
|
from vllm.model_executor.models.utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
WeightsMapper,
|
WeightsMapper,
|
||||||
@@ -35,6 +37,7 @@ from vllm.model_executor.models.utils import (
|
|||||||
maybe_prefix,
|
maybe_prefix,
|
||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||||
|
|
||||||
from .mm_preprocess import (
|
from .mm_preprocess import (
|
||||||
IMAGE_PLACEHOLDER,
|
IMAGE_PLACEHOLDER,
|
||||||
@@ -81,7 +84,7 @@ def _make_deepseek_v4_vl_weights_mapper(
|
|||||||
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
|
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
|
||||||
)
|
)
|
||||||
class DeepseekV4ForConditionalGeneration(
|
class DeepseekV4ForConditionalGeneration(
|
||||||
nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3
|
nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3, SupportsLoRA
|
||||||
):
|
):
|
||||||
"""Multimodal entry point for DeepSeek-V4 checkpoints with a vision tower.
|
"""Multimodal entry point for DeepSeek-V4 checkpoints with a vision tower.
|
||||||
|
|
||||||
@@ -89,9 +92,19 @@ class DeepseekV4ForConditionalGeneration(
|
|||||||
delegates through ``language_model`` via the protocol defaults.
|
delegates through ``language_model`` via the protocol defaults.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
packed_modules_mapping = {
|
||||||
|
"gate_up_proj": ["w1", "w3"],
|
||||||
|
"fused_wqa_wkv": ["wq_a", "wkv"],
|
||||||
|
"fused_wkv_wgate": ["wkv", "wgate"],
|
||||||
|
# for visual encoder
|
||||||
|
"wqkv": ["wqkv"],
|
||||||
|
"w1": ["w1"],
|
||||||
|
}
|
||||||
|
|
||||||
# The MoE router needs raw token ids to detect image sentinel tokens
|
# The MoE router needs raw token ids to detect image sentinel tokens
|
||||||
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
|
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
|
||||||
requires_raw_input_tokens = True
|
requires_raw_input_tokens = True
|
||||||
|
supports_tower_connector_lora = True
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||||
@@ -329,3 +342,29 @@ class DeepseekV4ForConditionalGeneration(
|
|||||||
return
|
return
|
||||||
self.language_model.process_weights_after_loading()
|
self.language_model.process_weights_after_loading()
|
||||||
self._weights_finalized = True
|
self._weights_finalized = True
|
||||||
|
|
||||||
|
def get_mm_mapping(self) -> MultiModelKeys:
|
||||||
|
"""Get the module prefixes in the multimodal model."""
|
||||||
|
return MultiModelKeys.from_string_field(
|
||||||
|
language_model="language_model",
|
||||||
|
connector="aligner",
|
||||||
|
tower_model="vision.",
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_mm_lora_token_counts(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
modality: str,
|
||||||
|
mm_kwargs: MultiModalKwargsItem | None,
|
||||||
|
num_mm_embeds: int,
|
||||||
|
) -> tuple[int, int | None]:
|
||||||
|
if modality != "image":
|
||||||
|
raise ValueError(f"Unsupported modality: {modality!r}")
|
||||||
|
|
||||||
|
patches = mm_kwargs.get("patches") if mm_kwargs else None
|
||||||
|
if patches is not None and isinstance(patches.data, torch.Tensor):
|
||||||
|
tower_tokens = patches.data.shape[0]
|
||||||
|
else:
|
||||||
|
tower_tokens = num_mm_embeds * self.config.vision_downsample_ratio**2
|
||||||
|
|
||||||
|
return tower_tokens, num_mm_embeds
|
||||||
|
|||||||
Reference in New Issue
Block a user