[New model][Multimodal] Add DeepSeek-V4-Flash-Vision-Exp support (#54566)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-02 11:09:09 -07:00
committed by GitHub
parent 488e6fd53c
commit 1356635d83
37 changed files with 2918 additions and 152 deletions
+257 -5
View File
@@ -26,18 +26,46 @@ def _torch_topk_softplus_sqrt(
e_score_correction_bias: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
):
scores = F.softplus(gating_output.float()).sqrt()
original_scores = scores
if e_score_correction_bias is not None:
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores
if hash_indices_table is not None:
assert input_ids is not None
topk_ids = hash_indices_table[input_ids.long()]
if bias_vl is None or image_sentinel_lo == 0:
topk_ids = hash_indices_table[input_ids.long()]
else:
image_mask = (input_ids.long() >= image_sentinel_lo) & (
input_ids.long() < image_sentinel_lo + 5
)
topk_ids = hash_indices_table[input_ids.long()]
vl_ids = (scores + bias_vl).topk(topk, dim=-1).indices
topk_ids = torch.where(
image_mask.unsqueeze(-1), vl_ids.to(topk_ids.dtype), topk_ids
)
else:
if bias_vl is not None and image_sentinel_lo > 0:
# Image tokens carry five consecutive in-vocab sentinel ids
# starting at image_sentinel_lo and select experts with bias_vl
# instead of the regular bias.
assert input_ids is not None
assert e_score_correction_bias is not None
image_mask = (
(input_ids.long() >= image_sentinel_lo)
& (input_ids.long() < image_sentinel_lo + 5)
).unsqueeze(-1)
row_bias = torch.where(
image_mask,
bias_vl.unsqueeze(0),
e_score_correction_bias.unsqueeze(0),
)
scores_for_choice = scores + row_bias
elif e_score_correction_bias is not None:
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores
# Match the fused kernel's deterministic tie-break: lower expert ids
# win when scores are equal. torch.topk does not guarantee which tied
# index it selects at the k-th boundary.
@@ -368,3 +396,227 @@ def test_fused_topk_softplus_sqrt_padding(
sorted_w_ref = topk_weights_ref[rows_to_compare].gather(1, idx_ref)
sorted_w = topk_weights[rows_to_compare].gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
def _make_mixed_input_ids(
num_tokens: int, image_sentinel_lo: int, dtype: torch.dtype = torch.long
) -> torch.Tensor:
"""Mix of text, image-sentinel, and above-sentinel ids.
Every third token is an image sentinel id (cycling through the five slots
[image_sentinel_lo, image_sentinel_lo + 5)); every sixth token carries an
id just above the sentinel block, simulating the named special tokens
that follow it — those must route as regular text tokens.
"""
input_ids = torch.randint(
0, image_sentinel_lo, (num_tokens,), dtype=torch.long, device="cuda"
)
pos = torch.arange(num_tokens, device="cuda")
image_rows = pos % 3 == 0
input_ids[image_rows] = image_sentinel_lo + (pos[image_rows] // 3) % 5
above_rows = pos % 6 == 3
input_ids[above_rows] = image_sentinel_lo + 5
return input_ids.to(dtype)
def _assert_topk_matches(
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights_ref: torch.Tensor,
topk_ids_ref: torch.Tensor,
) -> None:
# Different kernels may return the topk experts in different orders when
# scores tie; sort by expert id before comparing.
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(
sorted_ref_ids, sorted_ids.to(sorted_ref_ids.dtype), atol=0, rtol=0
)
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops.to(torch.long))
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize(
("num_experts", "topk", "renormalize", "dtype"),
[
(256, 6, True, torch.float32), # Triton dsv4 fast path
(256, 8, True, torch.bfloat16), # CUDA topk kernel
(384, 6, False, torch.half),
(512, 16, True, torch.float32),
],
)
def test_fused_topk_softplus_sqrt_bias_vl(
num_experts: int,
topk: int,
renormalize: bool,
dtype: torch.dtype,
):
"""Image sentinel tokens must select experts with bias_vl.
Rows with ids just above the sentinel block (image_sentinel_lo + 5)
simulate the named special tokens that follow it and must route through
the regular bias path; the reference comparison covers them.
"""
torch.manual_seed(0)
num_tokens = 64
hidden_size = 1024
vocab_size = 128
image_sentinel_lo = vocab_size - 8
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
e_score_correction_bias = torch.randn(
(num_experts,), dtype=torch.float32, device="cuda"
)
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
assert (input_ids == image_sentinel_lo + 5).any()
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=1.5,
e_score_correction_bias=e_score_correction_bias,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=e_score_correction_bias,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=1.5,
input_tokens=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize(
("num_experts", "topk", "renormalize", "dtype", "indices_dtype"),
[
(256, 6, True, torch.float32, torch.long), # specialized hash kernel
(384, 6, True, torch.float32, torch.int32), # specialized hash kernel
(384, 8, False, torch.bfloat16, torch.long), # generic hash kernel
(512, 6, True, torch.float32, torch.long), # generic hash kernel
],
)
def test_fused_topk_softplus_sqrt_hash_bias_vl(
num_experts: int,
topk: int,
renormalize: bool,
dtype: torch.dtype,
indices_dtype: torch.dtype,
):
"""Hash MoE: text rows use tid2eid, image rows use topk(score + bias_vl).
Sentinel ids are in-vocab; rows with ids above the sentinel block
(image_sentinel_lo + 5) must still come straight from the hash table.
"""
torch.manual_seed(0)
num_tokens = 64
hidden_size = 1024
vocab_size = 64
image_sentinel_lo = vocab_size - 8
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
hash_indices_table = torch.stack(
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
).to(device="cuda", dtype=indices_dtype)
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
image_rows = (input_ids >= image_sentinel_lo) & (input_ids < image_sentinel_lo + 5)
assert image_rows.any() and (~image_rows).any()
assert (input_ids == image_sentinel_lo + 5).any()
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=2.5,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=None,
topk=topk,
renormalize=renormalize,
input_tokens=input_ids,
hash_indices_table=hash_indices_table,
routed_scaling_factor=2.5,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
# Text rows must come straight from the hash table, image rows from
# topk(score + bias_vl).
text_ids = hash_indices_table[input_ids[~image_rows].long()]
torch.testing.assert_close(
topk_ids[~image_rows].to(text_ids.dtype), text_ids, atol=0, rtol=0
)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
def test_dsv4_fast_topk_bias_vl():
torch.manual_seed(0)
num_tokens = 33
num_experts = 256
vocab_size = 128
image_sentinel_lo = vocab_size - 8
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
bias_vl = torch.randn(num_experts, dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = dsv4_topk(
gating_output,
correction_bias,
torch.int64,
1.5,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
assert topk_ids.dtype == torch.int64
torch.testing.assert_close(topk_ids_ref.to(torch.int64), topk_ids, atol=0, rtol=0)
torch.testing.assert_close(topk_weights_ref, topk_weights, atol=2e-5, rtol=2e-5)