[New model][Multimodal] Add DeepSeek-V4-Flash-Vision-Exp support (#54566)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-02 11:09:09 -07:00
committed by GitHub
parent 488e6fd53c
commit 1356635d83
37 changed files with 2918 additions and 152 deletions
@@ -14,10 +14,39 @@ class DeepseekV4Config(PretrainedConfig):
rope_scaling: dict[str, Any] | None = None,
rope_parameters: dict[str, Any] | None = None,
rope_theta: float = 10000.0,
vision_n_layers: int = 0,
vision_dim: int = 1024,
vision_n_heads: int = 16,
vision_inter_dim: int = 2816,
vision_patch_size: int = 14,
vision_rope_theta: float = 10000.0,
vision_downsample_ratio: int = 3,
vision_max_n_token: int = 384,
vision_min_pixels: int = 147456,
vision_max_wh_ratio: float = 8,
**kwargs,
):
self.max_position_embeddings = max_position_embeddings
self.rope_scaling = rope_scaling
self.rope_theta = rope_theta
self.rope_parameters = rope_scaling or rope_parameters
# Vision tower config; vision_n_layers == 0 means text-only.
self.vision_n_layers = vision_n_layers
self.vision_dim = vision_dim
self.vision_n_heads = vision_n_heads
self.vision_inter_dim = vision_inter_dim
self.vision_patch_size = vision_patch_size
self.vision_rope_theta = vision_rope_theta
self.vision_downsample_ratio = vision_downsample_ratio
self.vision_max_n_token = vision_max_n_token
self.vision_min_pixels = vision_min_pixels
self.vision_max_wh_ratio = vision_max_wh_ratio
# The sparse-SWA index kernels widen the window within image spans
# in-kernel, so mm-prefix ranges longer than sliding_window must be
# kept (they are only consumed by those kernels).
self.mm_prefix_clamp_sliding_window = vision_n_layers > 0
# The visibility span covers the sentinel block [IMAGE_START,
# IMAGE_END]; the mm placeholder additionally carries a leading
# compressor-alignment pad (see mm_preprocess.COMPRESS_PAD_TO).
self.mm_prefix_span_leading_pad_modulus = 4 if vision_n_layers > 0 else 0
super().__init__(**kwargs)
@@ -579,6 +579,40 @@ class DeepSeekMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
class DeepseekV4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
def __init__(
self,
hf_config: PretrainedConfig,
hf_text_config: PretrainedConfig,
revision: str | None = None,
):
# DeepSeek-V4-Flash-Vision-Exp ships the same architectures/model_type
# as the text-only DeepSeek-V4-Flash; route to the VL wrapper class
# when the config carries a vision tower. Mutate (not just override
# get_architectures) because model-class resolution reads the raw
# hf_config.architectures (get_model_architecture).
# Only rewrite the stock text architecture: speculative-draft configs
# (DSparkDraftModel / DeepSeekV4MTPModel) keep their own classes, and
# the VL wrapper marks the config copy it hands to the inner text
# backbone with _dsv4_vl_inner so this rewrite does not recurse.
if (
getattr(hf_config, "vision_n_layers", 0) > 0
and getattr(hf_config, "architectures", None) == ["DeepseekV4ForCausalLM"]
and not getattr(hf_config, "_dsv4_vl_inner", False)
):
hf_config.architectures = ["DeepseekV4ForConditionalGeneration"]
super().__init__(hf_config, hf_text_config, revision)
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
# The vision variant needs the mm-prefix plumbing: it makes the
# scheduler prefill image spans atomically (disable_chunked_mm_input)
# and routes per-request image ranges to the sparse-SWA metadata
# builder, which widens the sliding window bidirectionally in-kernel.
if not supports_multimodal:
return False
return getattr(self.hf_config, "vision_n_layers", 0) > 0
class MimoMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
def get_num_hidden_layers(self) -> int:
return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
@@ -775,6 +809,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
"cohere_asr": CohereAsrModelArchConfigConvertor,
"dbrx": DbrxModelArchConfigConvertor,
"deepseek_mtp": DeepSeekMTPModelArchConfigConvertor,
"deepseek_v4": DeepseekV4ModelArchConfigConvertor,
"diffusion_gemma_text": Gemma4ModelArchConfigConvertor,
"ernie_mtp": ErnieMTPModelArchConfigConvertor,
"falcon": FalconModelArchConfigConvertor,