[Kernel][Perf] Support Qwen head ratios in fused GDN MTP (#52539)
Signed-off-by: kiroxu <148877251+BabyDrangoner@users.noreply.github.com>
This commit is contained in:
@@ -146,7 +146,7 @@ __device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) {
|
||||
return {x, y};
|
||||
}
|
||||
|
||||
template <typename StateT>
|
||||
template <typename StateT, int ValueHeadsPerKeyHead>
|
||||
__global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
const __nv_bfloat16* __restrict__ mixed_qkv,
|
||||
const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b,
|
||||
@@ -186,7 +186,7 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
return;
|
||||
}
|
||||
|
||||
const int key_head = value_head / 8;
|
||||
const int key_head = value_head / ValueHeadsPerKeyHead;
|
||||
__shared__ StateT shared_state[2][kChunkV][kDimK];
|
||||
__shared__ float shared_q[kMaxMtpTokens][kDimK];
|
||||
__shared__ float shared_k[kMaxMtpTokens][kDimK];
|
||||
@@ -370,7 +370,7 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename StateT>
|
||||
template <typename StateT, int ValueHeadsPerKeyHead>
|
||||
void launch_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log,
|
||||
torch::stable::Tensor const& dt_bias,
|
||||
@@ -395,17 +395,19 @@ void launch_gdn_decode_post_conv_mtp(
|
||||
get_current_cuda_stream(mixed_qkv.get_device_index());
|
||||
const int num_requests = static_cast<int>(state_indices.size(0));
|
||||
const dim3 grid(num_requests, num_value_heads);
|
||||
gdn_decode_post_conv_mtp_kernel<StateT><<<grid, kThreads, 0, stream>>>(
|
||||
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
|
||||
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
|
||||
static_cast<const int*>(state_indices.data_ptr()),
|
||||
static_cast<const int*>(cu_seqlens.data_ptr()),
|
||||
static_cast<const int*>(num_accepted_tokens.data_ptr()),
|
||||
static_cast<StateT*>(state.data_ptr()), output_gate,
|
||||
norm_weight.data_ptr(), static_cast<__nv_bfloat16*>(out.data_ptr()),
|
||||
num_key_heads, num_value_heads, static_cast<int>(state_indices.size(1)),
|
||||
dt_bias_type, norm_weight.scalar_type() == ScalarType::BFloat16,
|
||||
static_cast<float>(scale), static_cast<float>(norm_eps), strides);
|
||||
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead>
|
||||
<<<grid, kThreads, 0, stream>>>(
|
||||
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
|
||||
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
|
||||
static_cast<const int*>(state_indices.data_ptr()),
|
||||
static_cast<const int*>(cu_seqlens.data_ptr()),
|
||||
static_cast<const int*>(num_accepted_tokens.data_ptr()),
|
||||
static_cast<StateT*>(state.data_ptr()), output_gate,
|
||||
norm_weight.data_ptr(), static_cast<__nv_bfloat16*>(out.data_ptr()),
|
||||
num_key_heads, num_value_heads,
|
||||
static_cast<int>(state_indices.size(1)), dt_bias_type,
|
||||
norm_weight.scalar_type() == ScalarType::BFloat16,
|
||||
static_cast<float>(scale), static_cast<float>(norm_eps), strides);
|
||||
const cudaError_t error = cudaGetLastError();
|
||||
STD_TORCH_CHECK(error == cudaSuccess,
|
||||
"GDN decode MTP post-conv kernel launch failed: ",
|
||||
@@ -479,8 +481,12 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
STD_TORCH_CHECK(key_width > 0 && key_width % (2 * kDimK) == 0,
|
||||
"mixed_qkv width is inconsistent with state");
|
||||
const int num_key_heads = static_cast<int>(key_width / (2 * kDimK));
|
||||
STD_TORCH_CHECK(num_value_heads == 8 * num_key_heads,
|
||||
"GDN decode MTP fusion requires HV/H=8");
|
||||
const int value_heads_per_key_head = num_value_heads / num_key_heads;
|
||||
STD_TORCH_CHECK(
|
||||
num_value_heads % num_key_heads == 0 &&
|
||||
((value_heads_per_key_head >= 1 && value_heads_per_key_head <= 4) ||
|
||||
value_heads_per_key_head == 8),
|
||||
"GDN decode MTP fusion requires HV/H in {1, 2, 3, 4, 8}");
|
||||
|
||||
STD_TORCH_CHECK(state_indices.dim() == 2 && state_indices.size(0) > 0 &&
|
||||
state_indices.size(1) > 0 &&
|
||||
@@ -541,17 +547,35 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
const auto* b_ptr = static_cast<const __nv_bfloat16*>(b.data_ptr());
|
||||
const auto* output_gate_ptr =
|
||||
static_cast<const __nv_bfloat16*>(output_gate.data_ptr());
|
||||
if (state_scalar_type == ScalarType::Float) {
|
||||
launch_gdn_decode_post_conv_mtp<float>(
|
||||
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
|
||||
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
|
||||
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
|
||||
strides);
|
||||
} else {
|
||||
launch_gdn_decode_post_conv_mtp<__nv_bfloat16>(
|
||||
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead>() {
|
||||
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead>(
|
||||
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
|
||||
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
|
||||
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
|
||||
strides);
|
||||
};
|
||||
const auto dispatch_state_type = [&]<int ValueHeadsPerKeyHead>() {
|
||||
if (state_scalar_type == ScalarType::Float) {
|
||||
launch.template operator()<float, ValueHeadsPerKeyHead>();
|
||||
} else {
|
||||
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead>();
|
||||
}
|
||||
};
|
||||
switch (value_heads_per_key_head) {
|
||||
case 1:
|
||||
dispatch_state_type.template operator()<1>();
|
||||
break;
|
||||
case 2:
|
||||
dispatch_state_type.template operator()<2>();
|
||||
break;
|
||||
case 3:
|
||||
dispatch_state_type.template operator()<3>();
|
||||
break;
|
||||
case 4:
|
||||
dispatch_state_type.template operator()<4>();
|
||||
break;
|
||||
default:
|
||||
dispatch_state_type.template operator()<8>();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user