[New model][Multimodal] Add DeepSeek-V4-Flash-Vision-Exp support (#54566)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -131,6 +131,8 @@ def FusedMoEFactory(
|
||||
apply_routed_scale_to_output: bool = False,
|
||||
zero_expert_type: str | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
runner_cls: type[MoERunner] | None = None,
|
||||
runner_args: dict[str, Any] | None = None,
|
||||
routed_experts_cls: type[RoutedExperts] | None = None,
|
||||
@@ -199,6 +201,9 @@ def FusedMoEFactory(
|
||||
output instead of topk_weights
|
||||
zero_expert_type: Type of zero expert handling
|
||||
hash_indices_table: Hash table for expert indices
|
||||
bias_vl: Vision routing bias for image tokens (Deepseek V4)
|
||||
image_sentinel_lo: First of five consecutive in-vocab image sentinel
|
||||
ids (0 = vision routing disabled)
|
||||
runner_cls: Custom MoERunner class (None = use default MoERunner)
|
||||
runner_args: Additional arguments for runner constructor
|
||||
routed_experts_cls: Custom RoutedExperts class (None = use default)
|
||||
@@ -313,6 +318,8 @@ def FusedMoEFactory(
|
||||
zero_expert_type=zero_expert_type,
|
||||
num_logical_experts=logical_num_experts,
|
||||
hash_indices_table=hash_indices_table,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
if params_dtype is None:
|
||||
|
||||
@@ -44,8 +44,12 @@ if current_platform.is_cuda():
|
||||
topk_weights_ptr,
|
||||
topk_ids_ptr,
|
||||
routed_scaling_factor,
|
||||
input_ids_ptr,
|
||||
bias_vl_ptr,
|
||||
image_sentinel_lo,
|
||||
NUM_EXPERTS: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
HAS_VL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
@@ -54,6 +58,19 @@ if current_platform.is_cuda():
|
||||
bias = tl.load(
|
||||
correction_bias_ptr + expert_offsets, mask=expert_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
if HAS_VL:
|
||||
# Image tokens carry five consecutive in-vocab sentinel ids
|
||||
# starting at image_sentinel_lo and use bias_vl for expert
|
||||
# selection instead of the regular correction bias. Ids above the
|
||||
# sentinel block are regular special tokens and must not match.
|
||||
token_id = tl.load(input_ids_ptr + row).to(tl.int64)
|
||||
bias_vl = tl.load(
|
||||
bias_vl_ptr + expert_offsets, mask=expert_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
is_image = (token_id >= image_sentinel_lo) & (
|
||||
token_id < image_sentinel_lo + 5
|
||||
)
|
||||
bias = tl.where(is_image, bias_vl, bias)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
@@ -101,8 +118,17 @@ def dsv4_topk(
|
||||
correction_bias: torch.Tensor,
|
||||
indices_dtype: torch.dtype,
|
||||
routed_scaling_factor: float,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens, num_experts = gating_output.shape
|
||||
has_vl = bias_vl is not None and image_sentinel_lo > 0
|
||||
if bias_vl is not None:
|
||||
assert input_ids is not None, "bias_vl routing requires input_ids"
|
||||
assert bias_vl.dtype == torch.float32 and bias_vl.is_contiguous()
|
||||
assert bias_vl.shape == (num_experts,)
|
||||
assert input_ids.is_contiguous()
|
||||
shape = (num_tokens, _TOPK)
|
||||
topk_weights = gating_output.new_empty(shape, dtype=torch.float32)
|
||||
topk_ids = gating_output.new_empty(shape, dtype=indices_dtype)
|
||||
@@ -113,8 +139,12 @@ def dsv4_topk(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
routed_scaling_factor,
|
||||
input_ids,
|
||||
bias_vl,
|
||||
image_sentinel_lo,
|
||||
NUM_EXPERTS=num_experts,
|
||||
BLOCK_N=triton.next_power_of_2(num_experts),
|
||||
HAS_VL=has_vl,
|
||||
num_warps=1,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
|
||||
@@ -81,6 +81,8 @@ def vllm_topk_softplus_sqrt(
|
||||
input_tokens: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
ops.topk_hash_softplus_sqrt(
|
||||
topk_weights,
|
||||
@@ -93,6 +95,8 @@ def vllm_topk_softplus_sqrt(
|
||||
input_tokens,
|
||||
hash_indices_table,
|
||||
is_padding=_get_padding_mask(topk_indices.shape[0]),
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
return topk_weights, topk_indices
|
||||
@@ -122,6 +126,8 @@ def fused_topk_bias(
|
||||
input_tokens: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
):
|
||||
if (
|
||||
input_tokens is not None
|
||||
@@ -130,18 +136,28 @@ def fused_topk_bias(
|
||||
):
|
||||
input_tokens = input_tokens.to(dtype=hash_indices_table.dtype)
|
||||
|
||||
if bias_vl is not None:
|
||||
# Image tokens carry five consecutive in-vocab sentinel ids starting
|
||||
# at image_sentinel_lo and select experts with bias_vl instead of the
|
||||
# regular routing path. image_sentinel_lo == 0 disables this.
|
||||
assert input_tokens is not None, "bias_vl routing requires input_tokens"
|
||||
|
||||
if not rocm_aiter_ops.is_fused_moe_enabled():
|
||||
assert hidden_states.size(0) == gating_output.size(0), (
|
||||
"Number of tokens mismatch"
|
||||
)
|
||||
|
||||
output_indices_dtype = torch.int32 if indices_type is None else indices_type
|
||||
if scoring_func == "sqrtsoftplus" and can_use_dsv4_topk(
|
||||
gating_output,
|
||||
e_score_correction_bias,
|
||||
topk,
|
||||
renormalize,
|
||||
output_indices_dtype,
|
||||
if (
|
||||
scoring_func == "sqrtsoftplus"
|
||||
and hash_indices_table is None
|
||||
and can_use_dsv4_topk(
|
||||
gating_output,
|
||||
e_score_correction_bias,
|
||||
topk,
|
||||
renormalize,
|
||||
output_indices_dtype,
|
||||
)
|
||||
):
|
||||
assert e_score_correction_bias is not None
|
||||
return dsv4_topk(
|
||||
@@ -149,6 +165,9 @@ def fused_topk_bias(
|
||||
e_score_correction_bias,
|
||||
output_indices_dtype,
|
||||
routed_scaling_factor,
|
||||
input_ids=input_tokens,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
M, _ = hidden_states.size()
|
||||
@@ -200,6 +219,8 @@ def fused_topk_bias(
|
||||
input_tokens,
|
||||
hash_indices_table,
|
||||
routed_scaling_factor,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported scoring function: {scoring_func}")
|
||||
@@ -255,6 +276,8 @@ def fused_topk_bias(
|
||||
input_tokens,
|
||||
hash_indices_table,
|
||||
routed_scaling_factor,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
n_routed_experts = gating_output.shape[-1]
|
||||
@@ -270,10 +293,39 @@ def fused_topk_bias(
|
||||
) + e_score_correction_bias.unsqueeze(0)
|
||||
else:
|
||||
scores_for_choice = scores.view(-1, n_routed_experts)
|
||||
image_mask = None
|
||||
if bias_vl is not None and image_sentinel_lo > 0:
|
||||
# Image tokens (five consecutive sentinel ids starting at
|
||||
# image_sentinel_lo) select experts with bias_vl instead of
|
||||
# e_score_correction_bias / the hash table. Ids above the sentinel
|
||||
# block are regular special tokens and must not match.
|
||||
assert input_tokens is not None, "bias_vl routing requires input_tokens"
|
||||
image_mask = (
|
||||
(input_tokens >= image_sentinel_lo) & (input_tokens < image_sentinel_lo + 5)
|
||||
).unsqueeze(-1)
|
||||
text_bias = (
|
||||
e_score_correction_bias
|
||||
if e_score_correction_bias is not None
|
||||
else torch.zeros_like(bias_vl)
|
||||
)
|
||||
row_bias = torch.where(image_mask, bias_vl, text_bias)
|
||||
scores_for_choice = scores.view(-1, n_routed_experts) + row_bias
|
||||
# For batch invariance, use sorted=True to ensure deterministic expert selection
|
||||
if hash_indices_table is not None:
|
||||
assert input_tokens is not None
|
||||
topk_indices = hash_indices_table[input_tokens]
|
||||
if image_mask is None:
|
||||
topk_indices = hash_indices_table[input_tokens]
|
||||
else:
|
||||
# Clamp sentinel rows (overwritten below) to keep the table
|
||||
# lookup well-defined.
|
||||
safe_ids = torch.where(image_mask.squeeze(-1), 0, input_tokens).to(
|
||||
hash_indices_table.dtype
|
||||
)
|
||||
topk_indices = hash_indices_table[safe_ids]
|
||||
vl_indices = torch.topk(scores_for_choice, k=topk, dim=-1)[1]
|
||||
topk_indices = torch.where(
|
||||
image_mask, vl_indices.to(topk_indices.dtype), topk_indices
|
||||
)
|
||||
else:
|
||||
use_sorted = envs.VLLM_BATCH_INVARIANT
|
||||
topk_indices = torch.topk(scores_for_choice, k=topk, dim=-1, sorted=use_sorted)[
|
||||
@@ -306,6 +358,8 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
num_fused_shared_experts: int = 0,
|
||||
shared_expert_weight: float = 1.0,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
@@ -318,6 +372,11 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.scoring_func = scoring_func
|
||||
self._hash_indices_table = hash_indices_table
|
||||
# Vision bias: image sentinel tokens (five consecutive in-vocab ids
|
||||
# starting at image_sentinel_lo) select experts with bias_vl instead
|
||||
# of e_score_correction_bias / the hash table.
|
||||
self.bias_vl = bias_vl
|
||||
self.image_sentinel_lo = image_sentinel_lo
|
||||
# Fused shared experts: append constant slots (ids immediately after
|
||||
# the routed experts, [global, global+n)) routed to by every token at
|
||||
# ``shared_expert_weight``, AFTER the routed top-k is renormalized.
|
||||
@@ -357,6 +416,8 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=self._hash_indices_table,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
bias_vl=self.bias_vl.data if self.bias_vl is not None else None,
|
||||
image_sentinel_lo=self.image_sentinel_lo,
|
||||
)
|
||||
|
||||
if self.num_fused_shared_experts > 0:
|
||||
|
||||
@@ -60,6 +60,9 @@ def create_fused_moe_router(
|
||||
zero_expert_type: str | None = None,
|
||||
num_logical_experts: int | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
# Deepseek V4 vision routing bias parameters
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
) -> FusedMoERouter:
|
||||
"""
|
||||
Factory function to create the appropriate FusedMoERouter subclass based on
|
||||
@@ -211,6 +214,8 @@ def create_fused_moe_router(
|
||||
hash_indices_table=hash_indices_table,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
shared_expert_weight=shared_expert_weight,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
if (
|
||||
|
||||
@@ -1002,6 +1002,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
|
||||
"ColQwen3_5": ColQwen3_5Config,
|
||||
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV4ForConditionalGeneration": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
|
||||
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
|
||||
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
|
||||
|
||||
@@ -362,6 +362,10 @@ _MULTIMODAL_MODELS = {
|
||||
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
|
||||
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
|
||||
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
|
||||
"DeepseekV4ForConditionalGeneration": (
|
||||
"vllm.models.deepseek_v4",
|
||||
"DeepseekV4ForConditionalGeneration",
|
||||
),
|
||||
"Dots3NoteForCausalLM": (
|
||||
"vllm.models.dots3_note",
|
||||
"Dots3NoteForCausalLM",
|
||||
|
||||
Reference in New Issue
Block a user