[Kernel][Perf] Support Qwen head ratios in fused GDN MTP (#52539)

Signed-off-by: kiroxu <148877251+BabyDrangoner@users.noreply.github.com>
This commit is contained in:
kiroxu
2026-08-17 18:17:40 -07:00
committed by GitHub
parent c296851a7d
commit cdb8545a91
4 changed files with 142 additions and 34 deletions
@@ -146,7 +146,7 @@ __device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) {
return {x, y};
}
template <typename StateT>
template <typename StateT, int ValueHeadsPerKeyHead>
__global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
const __nv_bfloat16* __restrict__ mixed_qkv,
const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b,
@@ -186,7 +186,7 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
return;
}
const int key_head = value_head / 8;
const int key_head = value_head / ValueHeadsPerKeyHead;
__shared__ StateT shared_state[2][kChunkV][kDimK];
__shared__ float shared_q[kMaxMtpTokens][kDimK];
__shared__ float shared_k[kMaxMtpTokens][kDimK];
@@ -370,7 +370,7 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
}
}
template <typename StateT>
template <typename StateT, int ValueHeadsPerKeyHead>
void launch_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log,
torch::stable::Tensor const& dt_bias,
@@ -395,17 +395,19 @@ void launch_gdn_decode_post_conv_mtp(
get_current_cuda_stream(mixed_qkv.get_device_index());
const int num_requests = static_cast<int>(state_indices.size(0));
const dim3 grid(num_requests, num_value_heads);
gdn_decode_post_conv_mtp_kernel<StateT><<<grid, kThreads, 0, stream>>>(
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
static_cast<const int*>(state_indices.data_ptr()),
static_cast<const int*>(cu_seqlens.data_ptr()),
static_cast<const int*>(num_accepted_tokens.data_ptr()),
static_cast<StateT*>(state.data_ptr()), output_gate,
norm_weight.data_ptr(), static_cast<__nv_bfloat16*>(out.data_ptr()),
num_key_heads, num_value_heads, static_cast<int>(state_indices.size(1)),
dt_bias_type, norm_weight.scalar_type() == ScalarType::BFloat16,
static_cast<float>(scale), static_cast<float>(norm_eps), strides);
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead>
<<<grid, kThreads, 0, stream>>>(
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
static_cast<const int*>(state_indices.data_ptr()),
static_cast<const int*>(cu_seqlens.data_ptr()),
static_cast<const int*>(num_accepted_tokens.data_ptr()),
static_cast<StateT*>(state.data_ptr()), output_gate,
norm_weight.data_ptr(), static_cast<__nv_bfloat16*>(out.data_ptr()),
num_key_heads, num_value_heads,
static_cast<int>(state_indices.size(1)), dt_bias_type,
norm_weight.scalar_type() == ScalarType::BFloat16,
static_cast<float>(scale), static_cast<float>(norm_eps), strides);
const cudaError_t error = cudaGetLastError();
STD_TORCH_CHECK(error == cudaSuccess,
"GDN decode MTP post-conv kernel launch failed: ",
@@ -479,8 +481,12 @@ void fused_gdn_decode_post_conv_mtp(
STD_TORCH_CHECK(key_width > 0 && key_width % (2 * kDimK) == 0,
"mixed_qkv width is inconsistent with state");
const int num_key_heads = static_cast<int>(key_width / (2 * kDimK));
STD_TORCH_CHECK(num_value_heads == 8 * num_key_heads,
"GDN decode MTP fusion requires HV/H=8");
const int value_heads_per_key_head = num_value_heads / num_key_heads;
STD_TORCH_CHECK(
num_value_heads % num_key_heads == 0 &&
((value_heads_per_key_head >= 1 && value_heads_per_key_head <= 4) ||
value_heads_per_key_head == 8),
"GDN decode MTP fusion requires HV/H in {1, 2, 3, 4, 8}");
STD_TORCH_CHECK(state_indices.dim() == 2 && state_indices.size(0) > 0 &&
state_indices.size(1) > 0 &&
@@ -541,17 +547,35 @@ void fused_gdn_decode_post_conv_mtp(
const auto* b_ptr = static_cast<const __nv_bfloat16*>(b.data_ptr());
const auto* output_gate_ptr =
static_cast<const __nv_bfloat16*>(output_gate.data_ptr());
if (state_scalar_type == ScalarType::Float) {
launch_gdn_decode_post_conv_mtp<float>(
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
strides);
} else {
launch_gdn_decode_post_conv_mtp<__nv_bfloat16>(
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead>() {
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead>(
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
strides);
};
const auto dispatch_state_type = [&]<int ValueHeadsPerKeyHead>() {
if (state_scalar_type == ScalarType::Float) {
launch.template operator()<float, ValueHeadsPerKeyHead>();
} else {
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead>();
}
};
switch (value_heads_per_key_head) {
case 1:
dispatch_state_type.template operator()<1>();
break;
case 2:
dispatch_state_type.template operator()<2>();
break;
case 3:
dispatch_state_type.template operator()<3>();
break;
case 4:
dispatch_state_type.template operator()<4>();
break;
default:
dispatch_state_type.template operator()<8>();
break;
}
}