diff --git a/tests/kernels/moe/test_topk_softplus_sqrt.py b/tests/kernels/moe/test_topk_softplus_sqrt.py index a6f7f30805..0d92fe6eac 100644 --- a/tests/kernels/moe/test_topk_softplus_sqrt.py +++ b/tests/kernels/moe/test_topk_softplus_sqrt.py @@ -302,15 +302,18 @@ def test_fused_topk_softplus_sqrt_hash( @pytest.mark.parametrize( ("num_tokens", "num_experts", "indices_type"), [ - (0, 256, torch.uint32), - (17, 256, torch.uint32), + (0, 256, torch.int32), + (17, 256, torch.int32), (17, 384, torch.int64), ], ) +@pytest.mark.parametrize("use_padding", [False, True]) def test_dsv4_fast_topk( num_tokens: int, num_experts: int, indices_type: torch.dtype, + use_padding: bool, + monkeypatch: pytest.MonkeyPatch, ): torch.manual_seed(0) gating_output = torch.randn( @@ -325,8 +328,25 @@ def test_dsv4_fast_topk( routed_scaling_factor=1.5, e_score_correction_bias=correction_bias, ) - topk_weights, topk_ids = dsv4_topk( - gating_output, correction_bias, indices_type, 1.5 + is_padding = torch.arange(num_tokens, device="cuda") % 2 == 1 + if use_padding: + monkeypatch.setattr( + "vllm.model_executor.layers.fused_moe.router." + "fused_topk_bias_router._get_padding_mask", + lambda _: is_padding, + ) + gating_output[is_padding] = float("nan") + topk_weights_ref[is_padding] = 0 + topk_ids_ref[is_padding] = -1 + topk_weights, topk_ids = fused_topk_bias( + gating_output, + gating_output, + "sqrtsoftplus", + correction_bias, + 6, + True, + indices_type=indices_type, + routed_scaling_factor=1.5, ) assert topk_ids.dtype == indices_type @@ -338,6 +358,17 @@ def test_dsv4_fast_topk( rtol=2e-5, ) + if use_padding and num_tokens: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + weights, ids = dsv4_topk( + gating_output, correction_bias, indices_type, 1.5, is_padding=is_padding + ) + is_padding.fill_(True) + graph.replay() + assert (ids == -1).all() + assert (weights == 0).all() + @pytest.mark.skipif( not current_platform.is_cuda_alike(), diff --git a/tests/v1/spec_decode/test_dflash_prepare_inputs.py b/tests/v1/spec_decode/test_dflash_prepare_inputs.py index 3fc5caec2e..3ca466e78b 100644 --- a/tests/v1/spec_decode/test_dflash_prepare_inputs.py +++ b/tests/v1/spec_decode/test_dflash_prepare_inputs.py @@ -35,6 +35,7 @@ def _run_prepare( input_buffers = SimpleNamespace( input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device), positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device), + is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device), query_start_loc=torch.full( (max_num_reqs + 1,), -1, dtype=torch.int32, device=device ), @@ -177,6 +178,11 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2] assert out.temperature[2].item() == 1.0 assert out.seeds[2].item() == 17 + assert not out.input_buffers.is_padding[:3].any() + assert out.input_buffers.is_padding[3:].all() + assert (out.input_buffers.input_ids[3:] == 0).all() + assert (out.input_buffers.positions[3:] == 0).all() + assert (out.query_slot_mapping[3:] == PAD_SLOT_ID).all() def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp(): diff --git a/vllm/model_executor/layers/fused_moe/router/dsv4_topk.py b/vllm/model_executor/layers/fused_moe/router/dsv4_topk.py index dbd29f3d08..71cd605af7 100644 --- a/vllm/model_executor/layers/fused_moe/router/dsv4_topk.py +++ b/vllm/model_executor/layers/fused_moe/router/dsv4_topk.py @@ -31,7 +31,7 @@ def can_use_dsv4_topk( and correction_bias.is_contiguous() and topk == _TOPK and renormalize - and indices_dtype in (torch.int32, torch.uint32, torch.int64) + and indices_dtype in (torch.int32, torch.int64) ) @@ -41,6 +41,7 @@ if current_platform.is_cuda(): def _dsv4_topk_kernel( gating_output_ptr, correction_bias_ptr, + is_padding_ptr, topk_weights_ptr, topk_ids_ptr, routed_scaling_factor, @@ -49,6 +50,7 @@ if current_platform.is_cuda(): image_sentinel_lo, NUM_EXPERTS: tl.constexpr, BLOCK_N: tl.constexpr, + HAS_PADDING: tl.constexpr, HAS_VL: tl.constexpr, launch_pdl: tl.constexpr, ): @@ -106,6 +108,11 @@ if current_platform.is_cuda(): output_mask = topk_offsets < 6 output_offsets = row * 6 + topk_offsets + if HAS_PADDING: + is_padding = tl.load(is_padding_ptr + row) + selected_weights = tl.where(is_padding, 0.0, selected_weights) + selected_ids = tl.where(is_padding, -1, selected_ids) + if launch_pdl: tl.extra.cuda.gdc_launch_dependents() @@ -118,11 +125,20 @@ def dsv4_topk( correction_bias: torch.Tensor, indices_dtype: torch.dtype, routed_scaling_factor: float, + is_padding: torch.Tensor | None = None, input_ids: torch.Tensor | None = None, bias_vl: torch.Tensor | None = None, image_sentinel_lo: int = 0, ) -> tuple[torch.Tensor, torch.Tensor]: num_tokens, num_experts = gating_output.shape + if is_padding is not None: + assert is_padding.dtype == torch.bool + assert is_padding.shape == (num_tokens,) + assert is_padding.device == gating_output.device + assert is_padding.is_contiguous() + assert indices_dtype in (torch.int32, torch.int64), ( + "Padding requires a signed indices dtype for the -1 sentinel." + ) has_vl = bias_vl is not None and image_sentinel_lo > 0 if bias_vl is not None: assert input_ids is not None, "bias_vl routing requires input_ids" @@ -136,6 +152,7 @@ def dsv4_topk( _dsv4_topk_kernel[(num_tokens,)]( gating_output, correction_bias, + is_padding, topk_weights, topk_ids, routed_scaling_factor, @@ -144,6 +161,7 @@ def dsv4_topk( image_sentinel_lo, NUM_EXPERTS=num_experts, BLOCK_N=triton.next_power_of_2(num_experts), + HAS_PADDING=is_padding is not None, HAS_VL=has_vl, num_warps=1, launch_pdl=current_platform.is_arch_support_pdl(), diff --git a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py index 656b3cc585..0855d9dc00 100644 --- a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py +++ b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py @@ -176,6 +176,7 @@ def fused_topk_bias( ) output_indices_dtype = torch.int32 if indices_type is None else indices_type + padding_mask = _get_padding_mask(gating_output.shape[0]) if ( scoring_func == "sqrtsoftplus" and hash_indices_table is None @@ -193,6 +194,7 @@ def fused_topk_bias( e_score_correction_bias, output_indices_dtype, routed_scaling_factor, + is_padding=padding_mask, input_ids=input_tokens, bias_vl=bias_vl, image_sentinel_lo=image_sentinel_lo, diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index 5a13441ee8..790193dac5 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -271,6 +271,7 @@ class DFlashSpeculator(DraftModelSpeculator): num_tokens_across_dp=num_tokens_across_dp, slot_mapping=slot_mappings, batch_descriptor=batch_descriptor, + is_padding=self.input_buffers.is_padding[:num_tokens], ): last_hidden_states = self.model( input_ids=self.input_buffers.input_ids[:num_tokens], @@ -535,6 +536,7 @@ def _prepare_dflash_inputs_kernel( # Outputs out_input_ids_ptr, out_query_positions_ptr, + out_is_padding_ptr, out_query_start_loc_ptr, out_seq_lens_ptr, out_query_slot_mapping_ptr, @@ -665,6 +667,7 @@ def _prepare_dflash_inputs_kernel( ) tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query) + tl.store(out_is_padding_ptr + query_idx, False, mask=is_query) clamped_query_pos = tl.minimum(query_pos, max_model_len - 1) tl.store(out_query_positions_ptr + query_idx, clamped_query_pos, mask=is_query) tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query) @@ -727,6 +730,9 @@ def _prepare_dflash_inputs_kernel( for i in range(q_pad_start, max_num_tokens, BLOCK_SIZE): block = i + tl.arange(0, BLOCK_SIZE) mask = block < max_num_tokens + tl.store(out_input_ids_ptr + block, 0, mask=mask) + tl.store(out_query_positions_ptr + block, 0, mask=mask) + tl.store(out_is_padding_ptr + block, True, mask=mask) tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask) @@ -778,6 +784,7 @@ def prepare_dflash_inputs( _prepare_dflash_inputs_kernel[(num_reqs, num_blocks)]( input_buffers.input_ids, input_buffers.positions, + input_buffers.is_padding, input_buffers.query_start_loc, input_buffers.seq_lens, query_slot_mapping,