[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
@@ -4,6 +4,7 @@
*/
#include <cstdint>
#include <string>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
@@ -146,7 +147,7 @@ __device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) {
return {x, y};
}
template <typename StateT, int ValueHeadsPerKeyHead>
template <typename StateT, int ValueHeadsPerKeyHead, bool SigmoidGate>
__global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
const __nv_bfloat16* __restrict__ mixed_qkv,
const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b,
@@ -354,9 +355,11 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
#pragma unroll
for (int i = 0; i < 4; ++i) {
const int value = lane + i * 32;
const float gate = silu_fast(__bfloat162float(
const float gate_input = __bfloat162float(
output_gate[static_cast<int64_t>(token) * strides.gate_row +
value_head * kDimV + value]));
value_head * kDimV + value]);
const float gate =
SigmoidGate ? sigmoid_fast(gate_input) : silu_fast(gate_input);
const float weight =
norm_weight_is_bf16
? __bfloat162float(
@@ -370,7 +373,7 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
}
}
template <typename StateT, int ValueHeadsPerKeyHead>
template <typename StateT, int ValueHeadsPerKeyHead, bool SigmoidGate>
void launch_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log,
torch::stable::Tensor const& dt_bias,
@@ -395,7 +398,7 @@ void launch_gdn_decode_post_conv_mtp(
get_current_cuda_stream(mixed_qkv.get_device_index());
const int num_requests = static_cast<int>(state_indices.size(0));
const dim3 grid(num_requests, num_value_heads);
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead>
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead, SigmoidGate>
<<<grid, kThreads, 0, stream>>>(
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
@@ -425,7 +428,7 @@ void fused_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& num_accepted_tokens,
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
double scale, double norm_eps) {
double scale, double norm_eps, const std::string& output_gate_activation) {
using torch::headeronly::ScalarType;
STD_TORCH_CHECK(
@@ -466,6 +469,9 @@ void fused_gdn_decode_post_conv_mtp(
"norm_weight must be a CUDA float32 or bfloat16 tensor");
STD_TORCH_CHECK(out.is_cuda() && out.scalar_type() == ScalarType::BFloat16,
"out must be a CUDA bfloat16 tensor");
STD_TORCH_CHECK(
output_gate_activation == "silu" || output_gate_activation == "sigmoid",
"output_gate_activation must be 'silu' or 'sigmoid'");
STD_TORCH_CHECK(mixed_qkv.dim() == 2,
"mixed_qkv must have shape [L, 2 * H * 128 + HV * 128]");
@@ -547,8 +553,9 @@ void fused_gdn_decode_post_conv_mtp(
const auto* b_ptr = static_cast<const __nv_bfloat16*>(b.data_ptr());
const auto* output_gate_ptr =
static_cast<const __nv_bfloat16*>(output_gate.data_ptr());
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead>() {
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead>(
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead,
bool SigmoidGate>() {
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead, SigmoidGate>(
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
@@ -556,9 +563,18 @@ void fused_gdn_decode_post_conv_mtp(
};
const auto dispatch_state_type = [&]<int ValueHeadsPerKeyHead>() {
if (state_scalar_type == ScalarType::Float) {
launch.template operator()<float, ValueHeadsPerKeyHead>();
if (output_gate_activation == "sigmoid") {
launch.template operator()<float, ValueHeadsPerKeyHead, true>();
} else {
launch.template operator()<float, ValueHeadsPerKeyHead, false>();
}
} else {
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead>();
if (output_gate_activation == "sigmoid") {
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, true>();
} else {
launch
.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, false>();
}
}
};
switch (value_heads_per_key_head) {
+1 -1
View File
@@ -405,7 +405,7 @@ void fused_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& num_accepted_tokens,
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
double scale, double norm_eps);
double scale, double norm_eps, const std::string& output_gate_activation);
#endif
+2 -1
View File
@@ -535,7 +535,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"Tensor mixed_qkv, Tensor a, Tensor b, Tensor A_log, Tensor dt_bias, "
"Tensor state_indices, Tensor cu_seqlens, Tensor num_accepted_tokens, "
"Tensor! state, Tensor output_gate, Tensor norm_weight, Tensor! out, "
"float scale, float norm_eps=1e-5) -> ()");
"float scale, float norm_eps=1e-5, "
"str output_gate_activation='silu') -> ()");
#endif
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES