[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_runtime.h>
|
||||
@@ -146,7 +147,7 @@ __device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) {
|
||||
return {x, y};
|
||||
}
|
||||
|
||||
template <typename StateT, int ValueHeadsPerKeyHead>
|
||||
template <typename StateT, int ValueHeadsPerKeyHead, bool SigmoidGate>
|
||||
__global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
const __nv_bfloat16* __restrict__ mixed_qkv,
|
||||
const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b,
|
||||
@@ -354,9 +355,11 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
const int value = lane + i * 32;
|
||||
const float gate = silu_fast(__bfloat162float(
|
||||
const float gate_input = __bfloat162float(
|
||||
output_gate[static_cast<int64_t>(token) * strides.gate_row +
|
||||
value_head * kDimV + value]));
|
||||
value_head * kDimV + value]);
|
||||
const float gate =
|
||||
SigmoidGate ? sigmoid_fast(gate_input) : silu_fast(gate_input);
|
||||
const float weight =
|
||||
norm_weight_is_bf16
|
||||
? __bfloat162float(
|
||||
@@ -370,7 +373,7 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename StateT, int ValueHeadsPerKeyHead>
|
||||
template <typename StateT, int ValueHeadsPerKeyHead, bool SigmoidGate>
|
||||
void launch_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log,
|
||||
torch::stable::Tensor const& dt_bias,
|
||||
@@ -395,7 +398,7 @@ void launch_gdn_decode_post_conv_mtp(
|
||||
get_current_cuda_stream(mixed_qkv.get_device_index());
|
||||
const int num_requests = static_cast<int>(state_indices.size(0));
|
||||
const dim3 grid(num_requests, num_value_heads);
|
||||
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead>
|
||||
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead, SigmoidGate>
|
||||
<<<grid, kThreads, 0, stream>>>(
|
||||
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
|
||||
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
|
||||
@@ -425,7 +428,7 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& num_accepted_tokens,
|
||||
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
|
||||
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
|
||||
double scale, double norm_eps) {
|
||||
double scale, double norm_eps, const std::string& output_gate_activation) {
|
||||
using torch::headeronly::ScalarType;
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
@@ -466,6 +469,9 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
"norm_weight must be a CUDA float32 or bfloat16 tensor");
|
||||
STD_TORCH_CHECK(out.is_cuda() && out.scalar_type() == ScalarType::BFloat16,
|
||||
"out must be a CUDA bfloat16 tensor");
|
||||
STD_TORCH_CHECK(
|
||||
output_gate_activation == "silu" || output_gate_activation == "sigmoid",
|
||||
"output_gate_activation must be 'silu' or 'sigmoid'");
|
||||
|
||||
STD_TORCH_CHECK(mixed_qkv.dim() == 2,
|
||||
"mixed_qkv must have shape [L, 2 * H * 128 + HV * 128]");
|
||||
@@ -547,8 +553,9 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
const auto* b_ptr = static_cast<const __nv_bfloat16*>(b.data_ptr());
|
||||
const auto* output_gate_ptr =
|
||||
static_cast<const __nv_bfloat16*>(output_gate.data_ptr());
|
||||
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead>() {
|
||||
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead>(
|
||||
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead,
|
||||
bool SigmoidGate>() {
|
||||
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead, SigmoidGate>(
|
||||
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
|
||||
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
|
||||
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
|
||||
@@ -556,9 +563,18 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
};
|
||||
const auto dispatch_state_type = [&]<int ValueHeadsPerKeyHead>() {
|
||||
if (state_scalar_type == ScalarType::Float) {
|
||||
launch.template operator()<float, ValueHeadsPerKeyHead>();
|
||||
if (output_gate_activation == "sigmoid") {
|
||||
launch.template operator()<float, ValueHeadsPerKeyHead, true>();
|
||||
} else {
|
||||
launch.template operator()<float, ValueHeadsPerKeyHead, false>();
|
||||
}
|
||||
} else {
|
||||
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead>();
|
||||
if (output_gate_activation == "sigmoid") {
|
||||
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, true>();
|
||||
} else {
|
||||
launch
|
||||
.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, false>();
|
||||
}
|
||||
}
|
||||
};
|
||||
switch (value_heads_per_key_head) {
|
||||
|
||||
@@ -405,7 +405,7 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& num_accepted_tokens,
|
||||
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
|
||||
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
|
||||
double scale, double norm_eps);
|
||||
double scale, double norm_eps, const std::string& output_gate_activation);
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
@@ -535,7 +535,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor mixed_qkv, Tensor a, Tensor b, Tensor A_log, Tensor dt_bias, "
|
||||
"Tensor state_indices, Tensor cu_seqlens, Tensor num_accepted_tokens, "
|
||||
"Tensor! state, Tensor output_gate, Tensor norm_weight, Tensor! out, "
|
||||
"float scale, float norm_eps=1e-5) -> ()");
|
||||
"float scale, float norm_eps=1e-5, "
|
||||
"str output_gate_activation='silu') -> ()");
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
|
||||
Reference in New Issue
Block a user