[New model][Multimodal] Add DeepSeek-V4-Flash-Vision-Exp support (#54566)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-02 11:09:09 -07:00
committed by GitHub
parent 488e6fd53c
commit 1356635d83
37 changed files with 2918 additions and 152 deletions
@@ -131,6 +131,8 @@ def FusedMoEFactory(
apply_routed_scale_to_output: bool = False,
zero_expert_type: str | None = None,
hash_indices_table: torch.Tensor | None = None,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
runner_cls: type[MoERunner] | None = None,
runner_args: dict[str, Any] | None = None,
routed_experts_cls: type[RoutedExperts] | None = None,
@@ -199,6 +201,9 @@ def FusedMoEFactory(
output instead of topk_weights
zero_expert_type: Type of zero expert handling
hash_indices_table: Hash table for expert indices
bias_vl: Vision routing bias for image tokens (Deepseek V4)
image_sentinel_lo: First of five consecutive in-vocab image sentinel
ids (0 = vision routing disabled)
runner_cls: Custom MoERunner class (None = use default MoERunner)
runner_args: Additional arguments for runner constructor
routed_experts_cls: Custom RoutedExperts class (None = use default)
@@ -313,6 +318,8 @@ def FusedMoEFactory(
zero_expert_type=zero_expert_type,
num_logical_experts=logical_num_experts,
hash_indices_table=hash_indices_table,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
if params_dtype is None:
@@ -44,8 +44,12 @@ if current_platform.is_cuda():
topk_weights_ptr,
topk_ids_ptr,
routed_scaling_factor,
input_ids_ptr,
bias_vl_ptr,
image_sentinel_lo,
NUM_EXPERTS: tl.constexpr,
BLOCK_N: tl.constexpr,
HAS_VL: tl.constexpr,
launch_pdl: tl.constexpr,
):
row = tl.program_id(0)
@@ -54,6 +58,19 @@ if current_platform.is_cuda():
bias = tl.load(
correction_bias_ptr + expert_offsets, mask=expert_mask, other=0.0
).to(tl.float32)
if HAS_VL:
# Image tokens carry five consecutive in-vocab sentinel ids
# starting at image_sentinel_lo and use bias_vl for expert
# selection instead of the regular correction bias. Ids above the
# sentinel block are regular special tokens and must not match.
token_id = tl.load(input_ids_ptr + row).to(tl.int64)
bias_vl = tl.load(
bias_vl_ptr + expert_offsets, mask=expert_mask, other=0.0
).to(tl.float32)
is_image = (token_id >= image_sentinel_lo) & (
token_id < image_sentinel_lo + 5
)
bias = tl.where(is_image, bias_vl, bias)
if launch_pdl:
tl.extra.cuda.gdc_wait()
@@ -101,8 +118,17 @@ def dsv4_topk(
correction_bias: torch.Tensor,
indices_dtype: torch.dtype,
routed_scaling_factor: float,
input_ids: torch.Tensor | None = None,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens, num_experts = gating_output.shape
has_vl = bias_vl is not None and image_sentinel_lo > 0
if bias_vl is not None:
assert input_ids is not None, "bias_vl routing requires input_ids"
assert bias_vl.dtype == torch.float32 and bias_vl.is_contiguous()
assert bias_vl.shape == (num_experts,)
assert input_ids.is_contiguous()
shape = (num_tokens, _TOPK)
topk_weights = gating_output.new_empty(shape, dtype=torch.float32)
topk_ids = gating_output.new_empty(shape, dtype=indices_dtype)
@@ -113,8 +139,12 @@ def dsv4_topk(
topk_weights,
topk_ids,
routed_scaling_factor,
input_ids,
bias_vl,
image_sentinel_lo,
NUM_EXPERTS=num_experts,
BLOCK_N=triton.next_power_of_2(num_experts),
HAS_VL=has_vl,
num_warps=1,
launch_pdl=current_platform.is_arch_support_pdl(),
)
@@ -81,6 +81,8 @@ def vllm_topk_softplus_sqrt(
input_tokens: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
routed_scaling_factor: float = 1.0,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
) -> tuple[torch.Tensor, ...]:
ops.topk_hash_softplus_sqrt(
topk_weights,
@@ -93,6 +95,8 @@ def vllm_topk_softplus_sqrt(
input_tokens,
hash_indices_table,
is_padding=_get_padding_mask(topk_indices.shape[0]),
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
return topk_weights, topk_indices
@@ -122,6 +126,8 @@ def fused_topk_bias(
input_tokens: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
routed_scaling_factor: float = 1.0,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
):
if (
input_tokens is not None
@@ -130,18 +136,28 @@ def fused_topk_bias(
):
input_tokens = input_tokens.to(dtype=hash_indices_table.dtype)
if bias_vl is not None:
# Image tokens carry five consecutive in-vocab sentinel ids starting
# at image_sentinel_lo and select experts with bias_vl instead of the
# regular routing path. image_sentinel_lo == 0 disables this.
assert input_tokens is not None, "bias_vl routing requires input_tokens"
if not rocm_aiter_ops.is_fused_moe_enabled():
assert hidden_states.size(0) == gating_output.size(0), (
"Number of tokens mismatch"
)
output_indices_dtype = torch.int32 if indices_type is None else indices_type
if scoring_func == "sqrtsoftplus" and can_use_dsv4_topk(
gating_output,
e_score_correction_bias,
topk,
renormalize,
output_indices_dtype,
if (
scoring_func == "sqrtsoftplus"
and hash_indices_table is None
and can_use_dsv4_topk(
gating_output,
e_score_correction_bias,
topk,
renormalize,
output_indices_dtype,
)
):
assert e_score_correction_bias is not None
return dsv4_topk(
@@ -149,6 +165,9 @@ def fused_topk_bias(
e_score_correction_bias,
output_indices_dtype,
routed_scaling_factor,
input_ids=input_tokens,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
M, _ = hidden_states.size()
@@ -200,6 +219,8 @@ def fused_topk_bias(
input_tokens,
hash_indices_table,
routed_scaling_factor,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
else:
raise ValueError(f"Unsupported scoring function: {scoring_func}")
@@ -255,6 +276,8 @@ def fused_topk_bias(
input_tokens,
hash_indices_table,
routed_scaling_factor,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
n_routed_experts = gating_output.shape[-1]
@@ -270,10 +293,39 @@ def fused_topk_bias(
) + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores.view(-1, n_routed_experts)
image_mask = None
if bias_vl is not None and image_sentinel_lo > 0:
# Image tokens (five consecutive sentinel ids starting at
# image_sentinel_lo) select experts with bias_vl instead of
# e_score_correction_bias / the hash table. Ids above the sentinel
# block are regular special tokens and must not match.
assert input_tokens is not None, "bias_vl routing requires input_tokens"
image_mask = (
(input_tokens >= image_sentinel_lo) & (input_tokens < image_sentinel_lo + 5)
).unsqueeze(-1)
text_bias = (
e_score_correction_bias
if e_score_correction_bias is not None
else torch.zeros_like(bias_vl)
)
row_bias = torch.where(image_mask, bias_vl, text_bias)
scores_for_choice = scores.view(-1, n_routed_experts) + row_bias
# For batch invariance, use sorted=True to ensure deterministic expert selection
if hash_indices_table is not None:
assert input_tokens is not None
topk_indices = hash_indices_table[input_tokens]
if image_mask is None:
topk_indices = hash_indices_table[input_tokens]
else:
# Clamp sentinel rows (overwritten below) to keep the table
# lookup well-defined.
safe_ids = torch.where(image_mask.squeeze(-1), 0, input_tokens).to(
hash_indices_table.dtype
)
topk_indices = hash_indices_table[safe_ids]
vl_indices = torch.topk(scores_for_choice, k=topk, dim=-1)[1]
topk_indices = torch.where(
image_mask, vl_indices.to(topk_indices.dtype), topk_indices
)
else:
use_sorted = envs.VLLM_BATCH_INVARIANT
topk_indices = torch.topk(scores_for_choice, k=topk, dim=-1, sorted=use_sorted)[
@@ -306,6 +358,8 @@ class FusedTopKBiasRouter(BaseRouter):
hash_indices_table: torch.Tensor | None = None,
num_fused_shared_experts: int = 0,
shared_expert_weight: float = 1.0,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
):
super().__init__(
top_k=top_k,
@@ -318,6 +372,11 @@ class FusedTopKBiasRouter(BaseRouter):
self.routed_scaling_factor = routed_scaling_factor
self.scoring_func = scoring_func
self._hash_indices_table = hash_indices_table
# Vision bias: image sentinel tokens (five consecutive in-vocab ids
# starting at image_sentinel_lo) select experts with bias_vl instead
# of e_score_correction_bias / the hash table.
self.bias_vl = bias_vl
self.image_sentinel_lo = image_sentinel_lo
# Fused shared experts: append constant slots (ids immediately after
# the routed experts, [global, global+n)) routed to by every token at
# ``shared_expert_weight``, AFTER the routed top-k is renormalized.
@@ -357,6 +416,8 @@ class FusedTopKBiasRouter(BaseRouter):
input_tokens=input_ids,
hash_indices_table=self._hash_indices_table,
routed_scaling_factor=self.routed_scaling_factor,
bias_vl=self.bias_vl.data if self.bias_vl is not None else None,
image_sentinel_lo=self.image_sentinel_lo,
)
if self.num_fused_shared_experts > 0:
@@ -60,6 +60,9 @@ def create_fused_moe_router(
zero_expert_type: str | None = None,
num_logical_experts: int | None = None,
hash_indices_table: torch.Tensor | None = None,
# Deepseek V4 vision routing bias parameters
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
) -> FusedMoERouter:
"""
Factory function to create the appropriate FusedMoERouter subclass based on
@@ -211,6 +214,8 @@ def create_fused_moe_router(
hash_indices_table=hash_indices_table,
num_fused_shared_experts=num_fused_shared_experts,
shared_expert_weight=shared_expert_weight,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
if (
+1
View File
@@ -1002,6 +1002,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
"ColQwen3_5": ColQwen3_5Config,
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV4ForConditionalGeneration": DeepseekV4ForCausalLMConfig,
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
+4
View File
@@ -362,6 +362,10 @@ _MULTIMODAL_MODELS = {
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
"DeepseekV4ForConditionalGeneration": (
"vllm.models.deepseek_v4",
"DeepseekV4ForConditionalGeneration",
),
"Dots3NoteForCausalLM": (
"vllm.models.dots3_note",
"Dots3NoteForCausalLM",