[New model][Multimodal] Add DeepSeek-V4-Flash-Vision-Exp support (#54566)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -14,10 +14,39 @@ class DeepseekV4Config(PretrainedConfig):
|
||||
rope_scaling: dict[str, Any] | None = None,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
rope_theta: float = 10000.0,
|
||||
vision_n_layers: int = 0,
|
||||
vision_dim: int = 1024,
|
||||
vision_n_heads: int = 16,
|
||||
vision_inter_dim: int = 2816,
|
||||
vision_patch_size: int = 14,
|
||||
vision_rope_theta: float = 10000.0,
|
||||
vision_downsample_ratio: int = 3,
|
||||
vision_max_n_token: int = 384,
|
||||
vision_min_pixels: int = 147456,
|
||||
vision_max_wh_ratio: float = 8,
|
||||
**kwargs,
|
||||
):
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.rope_scaling = rope_scaling
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_parameters = rope_scaling or rope_parameters
|
||||
# Vision tower config; vision_n_layers == 0 means text-only.
|
||||
self.vision_n_layers = vision_n_layers
|
||||
self.vision_dim = vision_dim
|
||||
self.vision_n_heads = vision_n_heads
|
||||
self.vision_inter_dim = vision_inter_dim
|
||||
self.vision_patch_size = vision_patch_size
|
||||
self.vision_rope_theta = vision_rope_theta
|
||||
self.vision_downsample_ratio = vision_downsample_ratio
|
||||
self.vision_max_n_token = vision_max_n_token
|
||||
self.vision_min_pixels = vision_min_pixels
|
||||
self.vision_max_wh_ratio = vision_max_wh_ratio
|
||||
# The sparse-SWA index kernels widen the window within image spans
|
||||
# in-kernel, so mm-prefix ranges longer than sliding_window must be
|
||||
# kept (they are only consumed by those kernels).
|
||||
self.mm_prefix_clamp_sliding_window = vision_n_layers > 0
|
||||
# The visibility span covers the sentinel block [IMAGE_START,
|
||||
# IMAGE_END]; the mm placeholder additionally carries a leading
|
||||
# compressor-alignment pad (see mm_preprocess.COMPRESS_PAD_TO).
|
||||
self.mm_prefix_span_leading_pad_modulus = 4 if vision_n_layers > 0 else 0
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@@ -579,6 +579,40 @@ class DeepSeekMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
|
||||
|
||||
|
||||
class DeepseekV4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def __init__(
|
||||
self,
|
||||
hf_config: PretrainedConfig,
|
||||
hf_text_config: PretrainedConfig,
|
||||
revision: str | None = None,
|
||||
):
|
||||
# DeepSeek-V4-Flash-Vision-Exp ships the same architectures/model_type
|
||||
# as the text-only DeepSeek-V4-Flash; route to the VL wrapper class
|
||||
# when the config carries a vision tower. Mutate (not just override
|
||||
# get_architectures) because model-class resolution reads the raw
|
||||
# hf_config.architectures (get_model_architecture).
|
||||
# Only rewrite the stock text architecture: speculative-draft configs
|
||||
# (DSparkDraftModel / DeepSeekV4MTPModel) keep their own classes, and
|
||||
# the VL wrapper marks the config copy it hands to the inner text
|
||||
# backbone with _dsv4_vl_inner so this rewrite does not recurse.
|
||||
if (
|
||||
getattr(hf_config, "vision_n_layers", 0) > 0
|
||||
and getattr(hf_config, "architectures", None) == ["DeepseekV4ForCausalLM"]
|
||||
and not getattr(hf_config, "_dsv4_vl_inner", False)
|
||||
):
|
||||
hf_config.architectures = ["DeepseekV4ForConditionalGeneration"]
|
||||
super().__init__(hf_config, hf_text_config, revision)
|
||||
|
||||
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
|
||||
# The vision variant needs the mm-prefix plumbing: it makes the
|
||||
# scheduler prefill image spans atomically (disable_chunked_mm_input)
|
||||
# and routes per-request image ranges to the sparse-SWA metadata
|
||||
# builder, which widens the sliding window bidirectionally in-kernel.
|
||||
if not supports_multimodal:
|
||||
return False
|
||||
return getattr(self.hf_config, "vision_n_layers", 0) > 0
|
||||
|
||||
|
||||
class MimoMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def get_num_hidden_layers(self) -> int:
|
||||
return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
|
||||
@@ -775,6 +809,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
|
||||
"cohere_asr": CohereAsrModelArchConfigConvertor,
|
||||
"dbrx": DbrxModelArchConfigConvertor,
|
||||
"deepseek_mtp": DeepSeekMTPModelArchConfigConvertor,
|
||||
"deepseek_v4": DeepseekV4ModelArchConfigConvertor,
|
||||
"diffusion_gemma_text": Gemma4ModelArchConfigConvertor,
|
||||
"ernie_mtp": ErnieMTPModelArchConfigConvertor,
|
||||
"falcon": FalconModelArchConfigConvertor,
|
||||
|
||||
Reference in New Issue
Block a user