[Bugfix] Fix DSpark graph-padding handling in the DeepSeek-V4.1 MoE fused top-k (#59166)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chauncey
2026-10-14 07:49:52 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 791f92135a
commit ad8d39cbab
5 changed files with 69 additions and 5 deletions
+35 -4
View File
@@ -302,15 +302,18 @@ def test_fused_topk_softplus_sqrt_hash(
@pytest.mark.parametrize(
("num_tokens", "num_experts", "indices_type"),
[
(0, 256, torch.uint32),
(17, 256, torch.uint32),
(0, 256, torch.int32),
(17, 256, torch.int32),
(17, 384, torch.int64),
],
)
@pytest.mark.parametrize("use_padding", [False, True])
def test_dsv4_fast_topk(
num_tokens: int,
num_experts: int,
indices_type: torch.dtype,
use_padding: bool,
monkeypatch: pytest.MonkeyPatch,
):
torch.manual_seed(0)
gating_output = torch.randn(
@@ -325,8 +328,25 @@ def test_dsv4_fast_topk(
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
)
topk_weights, topk_ids = dsv4_topk(
gating_output, correction_bias, indices_type, 1.5
is_padding = torch.arange(num_tokens, device="cuda") % 2 == 1
if use_padding:
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.router."
"fused_topk_bias_router._get_padding_mask",
lambda _: is_padding,
)
gating_output[is_padding] = float("nan")
topk_weights_ref[is_padding] = 0
topk_ids_ref[is_padding] = -1
topk_weights, topk_ids = fused_topk_bias(
gating_output,
gating_output,
"sqrtsoftplus",
correction_bias,
6,
True,
indices_type=indices_type,
routed_scaling_factor=1.5,
)
assert topk_ids.dtype == indices_type
@@ -338,6 +358,17 @@ def test_dsv4_fast_topk(
rtol=2e-5,
)
if use_padding and num_tokens:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
weights, ids = dsv4_topk(
gating_output, correction_bias, indices_type, 1.5, is_padding=is_padding
)
is_padding.fill_(True)
graph.replay()
assert (ids == -1).all()
assert (weights == 0).all()
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
@@ -35,6 +35,7 @@ def _run_prepare(
input_buffers = SimpleNamespace(
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
query_start_loc=torch.full(
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
),
@@ -177,6 +178,11 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size
assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2]
assert out.temperature[2].item() == 1.0
assert out.seeds[2].item() == 17
assert not out.input_buffers.is_padding[:3].any()
assert out.input_buffers.is_padding[3:].all()
assert (out.input_buffers.input_ids[3:] == 0).all()
assert (out.input_buffers.positions[3:] == 0).all()
assert (out.query_slot_mapping[3:] == PAD_SLOT_ID).all()
def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp():
@@ -31,7 +31,7 @@ def can_use_dsv4_topk(
and correction_bias.is_contiguous()
and topk == _TOPK
and renormalize
and indices_dtype in (torch.int32, torch.uint32, torch.int64)
and indices_dtype in (torch.int32, torch.int64)
)
@@ -41,6 +41,7 @@ if current_platform.is_cuda():
def _dsv4_topk_kernel(
gating_output_ptr,
correction_bias_ptr,
is_padding_ptr,
topk_weights_ptr,
topk_ids_ptr,
routed_scaling_factor,
@@ -49,6 +50,7 @@ if current_platform.is_cuda():
image_sentinel_lo,
NUM_EXPERTS: tl.constexpr,
BLOCK_N: tl.constexpr,
HAS_PADDING: tl.constexpr,
HAS_VL: tl.constexpr,
launch_pdl: tl.constexpr,
):
@@ -106,6 +108,11 @@ if current_platform.is_cuda():
output_mask = topk_offsets < 6
output_offsets = row * 6 + topk_offsets
if HAS_PADDING:
is_padding = tl.load(is_padding_ptr + row)
selected_weights = tl.where(is_padding, 0.0, selected_weights)
selected_ids = tl.where(is_padding, -1, selected_ids)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
@@ -118,11 +125,20 @@ def dsv4_topk(
correction_bias: torch.Tensor,
indices_dtype: torch.dtype,
routed_scaling_factor: float,
is_padding: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens, num_experts = gating_output.shape
if is_padding is not None:
assert is_padding.dtype == torch.bool
assert is_padding.shape == (num_tokens,)
assert is_padding.device == gating_output.device
assert is_padding.is_contiguous()
assert indices_dtype in (torch.int32, torch.int64), (
"Padding requires a signed indices dtype for the -1 sentinel."
)
has_vl = bias_vl is not None and image_sentinel_lo > 0
if bias_vl is not None:
assert input_ids is not None, "bias_vl routing requires input_ids"
@@ -136,6 +152,7 @@ def dsv4_topk(
_dsv4_topk_kernel[(num_tokens,)](
gating_output,
correction_bias,
is_padding,
topk_weights,
topk_ids,
routed_scaling_factor,
@@ -144,6 +161,7 @@ def dsv4_topk(
image_sentinel_lo,
NUM_EXPERTS=num_experts,
BLOCK_N=triton.next_power_of_2(num_experts),
HAS_PADDING=is_padding is not None,
HAS_VL=has_vl,
num_warps=1,
launch_pdl=current_platform.is_arch_support_pdl(),
@@ -176,6 +176,7 @@ def fused_topk_bias(
)
output_indices_dtype = torch.int32 if indices_type is None else indices_type
padding_mask = _get_padding_mask(gating_output.shape[0])
if (
scoring_func == "sqrtsoftplus"
and hash_indices_table is None
@@ -193,6 +194,7 @@ def fused_topk_bias(
e_score_correction_bias,
output_indices_dtype,
routed_scaling_factor,
is_padding=padding_mask,
input_ids=input_tokens,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
@@ -271,6 +271,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
is_padding=self.input_buffers.is_padding[:num_tokens],
):
last_hidden_states = self.model(
input_ids=self.input_buffers.input_ids[:num_tokens],
@@ -535,6 +536,7 @@ def _prepare_dflash_inputs_kernel(
# Outputs
out_input_ids_ptr,
out_query_positions_ptr,
out_is_padding_ptr,
out_query_start_loc_ptr,
out_seq_lens_ptr,
out_query_slot_mapping_ptr,
@@ -665,6 +667,7 @@ def _prepare_dflash_inputs_kernel(
)
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
tl.store(out_is_padding_ptr + query_idx, False, mask=is_query)
clamped_query_pos = tl.minimum(query_pos, max_model_len - 1)
tl.store(out_query_positions_ptr + query_idx, clamped_query_pos, mask=is_query)
tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query)
@@ -727,6 +730,9 @@ def _prepare_dflash_inputs_kernel(
for i in range(q_pad_start, max_num_tokens, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_tokens
tl.store(out_input_ids_ptr + block, 0, mask=mask)
tl.store(out_query_positions_ptr + block, 0, mask=mask)
tl.store(out_is_padding_ptr + block, True, mask=mask)
tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask)
@@ -778,6 +784,7 @@ def prepare_dflash_inputs(
_prepare_dflash_inputs_kernel[(num_reqs, num_blocks)](
input_buffers.input_ids,
input_buffers.positions,
input_buffers.is_padding,
input_buffers.query_start_loc,
input_buffers.seq_lens,
query_slot_mapping,