[Perf] Optimize dsv4 routing using specialized kernel, 2.94% E2E TPOT improvement (#48660)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-07-17 13:35:06 -07:00
committed by GitHub
parent cc25f028b7
commit b5433b6f50
4 changed files with 264 additions and 0 deletions
@@ -71,6 +71,73 @@ __device__ __forceinline__ float toFloat(T value) {
}
}
#ifndef USE_ROCM
// Adapted from:
// https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/csrc/deepseek_v4/hash_topk.cuh
template <typename OutIndType, typename HashIndType>
__launch_bounds__(128) __global__
void dsv4HashTopkSoftplusSqrt(const float* input, float* output,
OutIndType* indices, int num_rows,
int num_experts, float routed_scaling_factor,
const HashIndType* input_ids,
const HashIndType* tid2eid) {
const int warp = (blockIdx.x * blockDim.x + threadIdx.x) / 32;
const int lane = threadIdx.x % 32;
if (warp >= num_rows) return;
const int64_t token_id = load_index_as_int64(input_ids, warp);
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaGridDependencySynchronize();
#endif
int expert = 0;
float weight = 0.f;
if (lane < 6) {
// only load and calculate for 6 experts
expert = static_cast<int>(tid2eid[token_id * 6 + lane]);
const float x = input[warp * num_experts + expert];
weight = sqrtf(fmaxf(x, 0.f) + __logf(1.f + __expf(-fabsf(x))));
}
float weight_sum = weight;
#pragma unroll
for (int mask = 16; mask > 0; mask >>= 1) {
// sum in warp
weight_sum += VLLM_SHFL_XOR_SYNC(weight_sum, mask);
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
#endif
if (lane < 6) {
const int offset = warp * 6 + lane;
output[offset] =
weight * routed_scaling_factor / (weight_sum > 0.f ? weight_sum : 1.f);
indices[offset] = static_cast<OutIndType>(expert);
}
}
template <typename OutIndType, typename HashIndType>
void launchDsv4HashTopk(const float* input, float* output, OutIndType* indices,
int num_rows, int num_experts,
double routed_scaling_factor,
const HashIndType* input_ids,
const HashIndType* tid2eid, cudaStream_t stream) {
if (num_rows == 0) return;
auto* kernel = &dsv4HashTopkSoftplusSqrt<OutIndType, HashIndType>;
cudaLaunchConfig_t config = {};
config.gridDim = (num_rows + 3) / 4;
config.blockDim = 128;
config.stream = stream;
cudaLaunchAttribute attr;
attr.id = cudaLaunchAttributeProgrammaticStreamSerialization;
attr.val.programmaticStreamSerializationAllowed = 1;
config.attrs = &attr;
config.numAttrs = 1;
const float scale = static_cast<float>(routed_scaling_factor);
cudaLaunchKernelEx(&config, kernel, input, output, indices, num_rows,
num_experts, scale, input_ids, tid2eid);
}
#endif
// ====================== TopK softplus_sqrt things
// ===============================
@@ -556,6 +623,17 @@ void topkGatingSoftplusSqrtKernelLauncher(
const float* correction_bias, const bool use_hash,
const HashIndType* input_ids, const HashIndType* tid2eid,
cudaStream_t stream) {
#ifndef USE_ROCM
if constexpr (std::is_same_v<InputType, float>) {
if (use_hash && topk == 6 && renormalize &&
(num_experts == 256 || num_experts == 384)) {
launchDsv4HashTopk<IndType, HashIndType>(
gating_output, topk_weights, topk_indices, num_tokens, num_experts,
routed_scaling_factor, input_ids, tid2eid, stream);
return;
}
}
#endif
static constexpr int WARPS_PER_TB = 4;
static constexpr int BYTES_PER_LDG_POWER_OF_2 = 16;
// for bfloat16 dtype, we need 4 bytes loading to make sure num_experts
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.router.dsv4_topk import dsv4_topk
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
fused_topk_bias,
)
@@ -186,3 +187,47 @@ def test_fused_topk_softplus_sqrt_hash(
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
@pytest.mark.parametrize(
("num_tokens", "num_experts", "indices_type"),
[
(0, 256, torch.uint32),
(17, 256, torch.uint32),
(17, 384, torch.int64),
],
)
def test_dsv4_fast_topk(
num_tokens: int,
num_experts: int,
indices_type: torch.dtype,
):
torch.manual_seed(0)
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
)
topk_weights, topk_ids = dsv4_topk(
gating_output, correction_bias, indices_type, 1.5
)
assert topk_ids.dtype == indices_type
torch.testing.assert_close(topk_ids_ref.to(indices_type), topk_ids, atol=0, rtol=0)
torch.testing.assert_close(
topk_weights_ref,
topk_weights,
atol=2e-5,
rtol=2e-5,
)
@@ -0,0 +1,121 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
_TOPK = 6
# Adapted from:
# https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/moe_fused_gate.py
def can_use_dsv4_topk(
gating_output: torch.Tensor,
correction_bias: torch.Tensor | None,
topk: int,
renormalize: bool,
indices_dtype: torch.dtype,
) -> bool:
return (
current_platform.is_cuda()
and gating_output.dtype == torch.float32
and gating_output.ndim == 2
and gating_output.shape[1] in (256, 384)
and gating_output.is_contiguous()
and correction_bias is not None
and correction_bias.dtype == torch.float32
and correction_bias.shape == (gating_output.shape[1],)
and correction_bias.is_contiguous()
and topk == _TOPK
and renormalize
and indices_dtype in (torch.int32, torch.uint32, torch.int64)
)
if current_platform.is_cuda():
@triton.jit
def _dsv4_topk_kernel(
gating_output_ptr,
correction_bias_ptr,
topk_weights_ptr,
topk_ids_ptr,
routed_scaling_factor,
NUM_EXPERTS: tl.constexpr,
BLOCK_N: tl.constexpr,
launch_pdl: tl.constexpr,
):
row = tl.program_id(0)
expert_offsets = tl.arange(0, BLOCK_N)
expert_mask = expert_offsets < NUM_EXPERTS
bias = tl.load(
correction_bias_ptr + expert_offsets, mask=expert_mask, other=0.0
).to(tl.float32)
if launch_pdl:
tl.extra.cuda.gdc_wait()
logits = tl.load(
gating_output_ptr + row * NUM_EXPERTS + expert_offsets,
mask=expert_mask,
other=0.0,
).to(tl.float32)
weights = tl.sqrt(tl.where(logits > 20.0, logits, tl.log(1.0 + tl.exp(logits))))
current = tl.where(expert_mask, weights + bias, -float("inf"))
current = tl.where(current == current, current, -1e30)
topk_offsets = tl.arange(0, 8)
selected_weights = tl.zeros([8], dtype=tl.float32)
selected_ids = tl.zeros([8], dtype=tl.int32)
for slot in tl.static_range(6):
max_value = tl.max(current, axis=0)
candidate = tl.where(current == max_value, expert_offsets, NUM_EXPERTS)
expert_id = tl.min(candidate, axis=0).to(tl.int32)
selected_weight = tl.sum(
tl.where(expert_offsets == expert_id, weights, 0.0), axis=0
)
is_slot = topk_offsets == slot
selected_weights = tl.where(is_slot, selected_weight, selected_weights)
selected_ids = tl.where(is_slot, expert_id, selected_ids)
current = tl.where(expert_offsets == expert_id, -float("inf"), current)
weight_sum = tl.sum(selected_weights, axis=0)
selected_weights *= routed_scaling_factor / tl.where(
weight_sum > 0.0, weight_sum, 1.0
)
output_mask = topk_offsets < 6
output_offsets = row * 6 + topk_offsets
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(topk_weights_ptr + output_offsets, selected_weights, mask=output_mask)
tl.store(topk_ids_ptr + output_offsets, selected_ids, mask=output_mask)
def dsv4_topk(
gating_output: torch.Tensor,
correction_bias: torch.Tensor,
indices_dtype: torch.dtype,
routed_scaling_factor: float,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens, num_experts = gating_output.shape
shape = (num_tokens, _TOPK)
topk_weights = gating_output.new_empty(shape, dtype=torch.float32)
topk_ids = gating_output.new_empty(shape, dtype=indices_dtype)
if num_tokens > 0:
_dsv4_topk_kernel[(num_tokens,)](
gating_output,
correction_bias,
topk_weights,
topk_ids,
routed_scaling_factor,
NUM_EXPERTS=num_experts,
BLOCK_N=triton.next_power_of_2(num_experts),
num_warps=1,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return topk_weights, topk_ids
@@ -14,6 +14,10 @@ from vllm.model_executor.layers.fused_moe.config import (
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
from vllm.model_executor.layers.fused_moe.router.dsv4_topk import (
can_use_dsv4_topk,
dsv4_topk,
)
def vllm_topk_softmax(
@@ -182,6 +186,22 @@ def fused_topk_bias(
"Number of tokens mismatch"
)
output_indices_dtype = torch.int32 if indices_type is None else indices_type
if scoring_func == "sqrtsoftplus" and can_use_dsv4_topk(
gating_output,
e_score_correction_bias,
topk,
renormalize,
output_indices_dtype,
):
assert e_score_correction_bias is not None
return dsv4_topk(
gating_output,
e_score_correction_bias,
output_indices_dtype,
routed_scaling_factor,
)
M, _ = hidden_states.size()
topk_weights = torch.empty(