[New model][Multimodal] Add DeepSeek-V4-Flash-Vision-Exp support (#54566)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -26,18 +26,46 @@ def _torch_topk_softplus_sqrt(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
):
|
||||
scores = F.softplus(gating_output.float()).sqrt()
|
||||
original_scores = scores
|
||||
if e_score_correction_bias is not None:
|
||||
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
|
||||
else:
|
||||
scores_for_choice = scores
|
||||
|
||||
if hash_indices_table is not None:
|
||||
assert input_ids is not None
|
||||
topk_ids = hash_indices_table[input_ids.long()]
|
||||
if bias_vl is None or image_sentinel_lo == 0:
|
||||
topk_ids = hash_indices_table[input_ids.long()]
|
||||
else:
|
||||
image_mask = (input_ids.long() >= image_sentinel_lo) & (
|
||||
input_ids.long() < image_sentinel_lo + 5
|
||||
)
|
||||
topk_ids = hash_indices_table[input_ids.long()]
|
||||
vl_ids = (scores + bias_vl).topk(topk, dim=-1).indices
|
||||
topk_ids = torch.where(
|
||||
image_mask.unsqueeze(-1), vl_ids.to(topk_ids.dtype), topk_ids
|
||||
)
|
||||
else:
|
||||
if bias_vl is not None and image_sentinel_lo > 0:
|
||||
# Image tokens carry five consecutive in-vocab sentinel ids
|
||||
# starting at image_sentinel_lo and select experts with bias_vl
|
||||
# instead of the regular bias.
|
||||
assert input_ids is not None
|
||||
assert e_score_correction_bias is not None
|
||||
image_mask = (
|
||||
(input_ids.long() >= image_sentinel_lo)
|
||||
& (input_ids.long() < image_sentinel_lo + 5)
|
||||
).unsqueeze(-1)
|
||||
row_bias = torch.where(
|
||||
image_mask,
|
||||
bias_vl.unsqueeze(0),
|
||||
e_score_correction_bias.unsqueeze(0),
|
||||
)
|
||||
scores_for_choice = scores + row_bias
|
||||
elif e_score_correction_bias is not None:
|
||||
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
|
||||
else:
|
||||
scores_for_choice = scores
|
||||
# Match the fused kernel's deterministic tie-break: lower expert ids
|
||||
# win when scores are equal. torch.topk does not guarantee which tied
|
||||
# index it selects at the k-th boundary.
|
||||
@@ -368,3 +396,227 @@ def test_fused_topk_softplus_sqrt_padding(
|
||||
sorted_w_ref = topk_weights_ref[rows_to_compare].gather(1, idx_ref)
|
||||
sorted_w = topk_weights[rows_to_compare].gather(1, idx_ops)
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _make_mixed_input_ids(
|
||||
num_tokens: int, image_sentinel_lo: int, dtype: torch.dtype = torch.long
|
||||
) -> torch.Tensor:
|
||||
"""Mix of text, image-sentinel, and above-sentinel ids.
|
||||
|
||||
Every third token is an image sentinel id (cycling through the five slots
|
||||
[image_sentinel_lo, image_sentinel_lo + 5)); every sixth token carries an
|
||||
id just above the sentinel block, simulating the named special tokens
|
||||
that follow it — those must route as regular text tokens.
|
||||
"""
|
||||
input_ids = torch.randint(
|
||||
0, image_sentinel_lo, (num_tokens,), dtype=torch.long, device="cuda"
|
||||
)
|
||||
pos = torch.arange(num_tokens, device="cuda")
|
||||
image_rows = pos % 3 == 0
|
||||
input_ids[image_rows] = image_sentinel_lo + (pos[image_rows] // 3) % 5
|
||||
above_rows = pos % 6 == 3
|
||||
input_ids[above_rows] = image_sentinel_lo + 5
|
||||
return input_ids.to(dtype)
|
||||
|
||||
|
||||
def _assert_topk_matches(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights_ref: torch.Tensor,
|
||||
topk_ids_ref: torch.Tensor,
|
||||
) -> None:
|
||||
# Different kernels may return the topk experts in different orders when
|
||||
# scores tie; sort by expert id before comparing.
|
||||
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
|
||||
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
|
||||
torch.testing.assert_close(
|
||||
sorted_ref_ids, sorted_ids.to(sorted_ref_ids.dtype), atol=0, rtol=0
|
||||
)
|
||||
|
||||
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
|
||||
sorted_w = topk_weights.gather(1, idx_ops.to(torch.long))
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "topk", "renormalize", "dtype"),
|
||||
[
|
||||
(256, 6, True, torch.float32), # Triton dsv4 fast path
|
||||
(256, 8, True, torch.bfloat16), # CUDA topk kernel
|
||||
(384, 6, False, torch.half),
|
||||
(512, 16, True, torch.float32),
|
||||
],
|
||||
)
|
||||
def test_fused_topk_softplus_sqrt_bias_vl(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
"""Image sentinel tokens must select experts with bias_vl.
|
||||
|
||||
Rows with ids just above the sentinel block (image_sentinel_lo + 5)
|
||||
simulate the named special tokens that follow it and must route through
|
||||
the regular bias path; the reference comparison covers them.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 64
|
||||
hidden_size = 1024
|
||||
vocab_size = 128
|
||||
image_sentinel_lo = vocab_size - 8
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
e_score_correction_bias = torch.randn(
|
||||
(num_experts,), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
|
||||
assert (input_ids == image_sentinel_lo + 5).any()
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
input_ids=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=1.5,
|
||||
input_tokens=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "topk", "renormalize", "dtype", "indices_dtype"),
|
||||
[
|
||||
(256, 6, True, torch.float32, torch.long), # specialized hash kernel
|
||||
(384, 6, True, torch.float32, torch.int32), # specialized hash kernel
|
||||
(384, 8, False, torch.bfloat16, torch.long), # generic hash kernel
|
||||
(512, 6, True, torch.float32, torch.long), # generic hash kernel
|
||||
],
|
||||
)
|
||||
def test_fused_topk_softplus_sqrt_hash_bias_vl(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
dtype: torch.dtype,
|
||||
indices_dtype: torch.dtype,
|
||||
):
|
||||
"""Hash MoE: text rows use tid2eid, image rows use topk(score + bias_vl).
|
||||
|
||||
Sentinel ids are in-vocab; rows with ids above the sentinel block
|
||||
(image_sentinel_lo + 5) must still come straight from the hash table.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 64
|
||||
hidden_size = 1024
|
||||
vocab_size = 64
|
||||
image_sentinel_lo = vocab_size - 8
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
hash_indices_table = torch.stack(
|
||||
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
|
||||
).to(device="cuda", dtype=indices_dtype)
|
||||
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
|
||||
image_rows = (input_ids >= image_sentinel_lo) & (input_ids < image_sentinel_lo + 5)
|
||||
assert image_rows.any() and (~image_rows).any()
|
||||
assert (input_ids == image_sentinel_lo + 5).any()
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=2.5,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=None,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
routed_scaling_factor=2.5,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
|
||||
|
||||
# Text rows must come straight from the hash table, image rows from
|
||||
# topk(score + bias_vl).
|
||||
text_ids = hash_indices_table[input_ids[~image_rows].long()]
|
||||
torch.testing.assert_close(
|
||||
topk_ids[~image_rows].to(text_ids.dtype), text_ids, atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="The DeepSeek V4 fast path is CUDA-only.",
|
||||
)
|
||||
def test_dsv4_fast_topk_bias_vl():
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 33
|
||||
num_experts = 256
|
||||
vocab_size = 128
|
||||
image_sentinel_lo = vocab_size - 8
|
||||
gating_output = torch.randn(
|
||||
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
|
||||
bias_vl = torch.randn(num_experts, dtype=torch.float32, device="cuda")
|
||||
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=6,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=correction_bias,
|
||||
input_ids=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
topk_weights, topk_ids = dsv4_topk(
|
||||
gating_output,
|
||||
correction_bias,
|
||||
torch.int64,
|
||||
1.5,
|
||||
input_ids=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
assert topk_ids.dtype == torch.int64
|
||||
torch.testing.assert_close(topk_ids_ref.to(torch.int64), topk_ids, atol=0, rtol=0)
|
||||
torch.testing.assert_close(topk_weights_ref, topk_weights, atol=2e-5, rtol=2e-5)
|
||||
|
||||
Reference in New Issue
Block a user