[Bugfix] Fix DSpark graph-padding handling in the DeepSeek-V4.1 MoE fused top-k (#59166)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
791f92135a
commit
ad8d39cbab
@@ -302,15 +302,18 @@ def test_fused_topk_softplus_sqrt_hash(
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "num_experts", "indices_type"),
|
||||
[
|
||||
(0, 256, torch.uint32),
|
||||
(17, 256, torch.uint32),
|
||||
(0, 256, torch.int32),
|
||||
(17, 256, torch.int32),
|
||||
(17, 384, torch.int64),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("use_padding", [False, True])
|
||||
def test_dsv4_fast_topk(
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
indices_type: torch.dtype,
|
||||
use_padding: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
gating_output = torch.randn(
|
||||
@@ -325,8 +328,25 @@ def test_dsv4_fast_topk(
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=correction_bias,
|
||||
)
|
||||
topk_weights, topk_ids = dsv4_topk(
|
||||
gating_output, correction_bias, indices_type, 1.5
|
||||
is_padding = torch.arange(num_tokens, device="cuda") % 2 == 1
|
||||
if use_padding:
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.router."
|
||||
"fused_topk_bias_router._get_padding_mask",
|
||||
lambda _: is_padding,
|
||||
)
|
||||
gating_output[is_padding] = float("nan")
|
||||
topk_weights_ref[is_padding] = 0
|
||||
topk_ids_ref[is_padding] = -1
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
gating_output,
|
||||
gating_output,
|
||||
"sqrtsoftplus",
|
||||
correction_bias,
|
||||
6,
|
||||
True,
|
||||
indices_type=indices_type,
|
||||
routed_scaling_factor=1.5,
|
||||
)
|
||||
|
||||
assert topk_ids.dtype == indices_type
|
||||
@@ -338,6 +358,17 @@ def test_dsv4_fast_topk(
|
||||
rtol=2e-5,
|
||||
)
|
||||
|
||||
if use_padding and num_tokens:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
weights, ids = dsv4_topk(
|
||||
gating_output, correction_bias, indices_type, 1.5, is_padding=is_padding
|
||||
)
|
||||
is_padding.fill_(True)
|
||||
graph.replay()
|
||||
assert (ids == -1).all()
|
||||
assert (weights == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
|
||||
@@ -35,6 +35,7 @@ def _run_prepare(
|
||||
input_buffers = SimpleNamespace(
|
||||
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
|
||||
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
|
||||
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
|
||||
query_start_loc=torch.full(
|
||||
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
|
||||
),
|
||||
@@ -177,6 +178,11 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size
|
||||
assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2]
|
||||
assert out.temperature[2].item() == 1.0
|
||||
assert out.seeds[2].item() == 17
|
||||
assert not out.input_buffers.is_padding[:3].any()
|
||||
assert out.input_buffers.is_padding[3:].all()
|
||||
assert (out.input_buffers.input_ids[3:] == 0).all()
|
||||
assert (out.input_buffers.positions[3:] == 0).all()
|
||||
assert (out.query_slot_mapping[3:] == PAD_SLOT_ID).all()
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp():
|
||||
|
||||
Reference in New Issue
Block a user