[Kernel][Perf] Add fused CUDA post-conv MTP decode kernel for Qwen3.5 GDN (#51674)

Signed-off-by: Jie Fang <jief@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jie Fang
2026-08-14 11:44:45 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Fable 5
parent d18bb7b390
commit 1be3628367
12 changed files with 1456 additions and 7 deletions
+18
View File
@@ -1118,6 +1118,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
cuda_archs_loose_intersection(FUSED_KDA_DECODE_ARCHS
"9.0a;10.0f;12.0f" "${CUDA_ARCHS}")
cuda_archs_loose_intersection(FUSED_GDN_DECODE_ARCHS
"8.0;8.6;8.9;9.0a;10.0f;12.0f" "${CUDA_ARCHS}")
endif()
if(FUSED_KDA_DECODE_ARCHS)
set(FUSED_KDA_DECODE_SRC
@@ -1131,6 +1133,18 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
message(STATUS
"Building fused KDA decode for archs: ${FUSED_KDA_DECODE_ARCHS}")
endif()
if(FUSED_GDN_DECODE_ARCHS)
set(FUSED_GDN_DECODE_SRC
"csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu")
set_gencode_flags_for_srcs(
SRCS "${FUSED_GDN_DECODE_SRC}"
CUDA_ARCHS "${FUSED_GDN_DECODE_ARCHS}")
set_property(SOURCE ${FUSED_GDN_DECODE_SRC} APPEND PROPERTY
COMPILE_OPTIONS "$<$<COMPILE_LANGUAGE:CUDA>:--use_fast_math>")
list(APPEND VLLM_STABLE_EXT_SRC "${FUSED_GDN_DECODE_SRC}")
message(STATUS
"Building fused GDN decode for archs: ${FUSED_GDN_DECODE_ARCHS}")
endif()
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
cuda_archs_loose_intersection(KIMI_K3_ATTN_RES_ARCHS
@@ -1213,6 +1227,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
target_compile_definitions(_C_stable_libtorch PRIVATE
VLLM_ENABLE_FUSED_KDA_DECODE=1)
endif()
if(FUSED_GDN_DECODE_ARCHS)
target_compile_definitions(_C_stable_libtorch PRIVATE
VLLM_ENABLE_FUSED_GDN_DECODE=1)
endif()
if(KIMI_K3_ATTN_RES_ARCHS)
target_compile_definitions(_C_stable_libtorch PRIVATE
VLLM_ENABLE_KIMI_K3_ATTN_RES=1)
@@ -0,0 +1,557 @@
/*
* SPDX-License-Identifier: Apache-2.0
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
*/
#include <cstdint>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include "../torch_utils.h"
#include "../../cuda_compat.h"
namespace {
template <typename StateT>
__device__ __forceinline__ void cp_async_16b(StateT* smem_ptr,
const StateT* gmem_ptr) {
const uint32_t smem_addr =
static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n"
:
: "r"(smem_addr), "l"(gmem_ptr));
}
__device__ __forceinline__ void cp_async_commit() {
asm volatile("cp.async.commit_group;\n" ::);
}
__device__ __forceinline__ void cp_async_wait_all() {
asm volatile("cp.async.wait_all;\n" ::: "memory");
}
template <typename StateT, int ChunkV, int DimK, int Stages>
__device__ __forceinline__ void copy_state_chunk(StateT* shared_state,
const StateT* state, int chunk,
int thread, int threads) {
constexpr int kElementsPerCopy = 16 / sizeof(StateT);
constexpr int kCopiesPerChunk = ChunkV * DimK / kElementsPerCopy;
const int stage = chunk % Stages;
for (int copy = thread; copy < kCopiesPerChunk; copy += threads) {
const int element = copy * kElementsPerCopy;
cp_async_16b(shared_state + stage * ChunkV * DimK + element,
state + chunk * ChunkV * DimK + element);
}
cp_async_commit();
}
template <typename StateT>
__device__ __forceinline__ float4 load_state4(const StateT* state);
template <>
__device__ __forceinline__ float4 load_state4<float>(const float* state) {
return *reinterpret_cast<const float4*>(state);
}
template <>
__device__ __forceinline__ float4
load_state4<__nv_bfloat16>(const __nv_bfloat16* state) {
const __nv_bfloat162 lo = *reinterpret_cast<const __nv_bfloat162*>(state);
const __nv_bfloat162 hi = *reinterpret_cast<const __nv_bfloat162*>(state + 2);
return make_float4(__bfloat162float(lo.x), __bfloat162float(lo.y),
__bfloat162float(hi.x), __bfloat162float(hi.y));
}
template <typename StateT>
__device__ __forceinline__ void store_state4(StateT* state, float4 value);
template <>
__device__ __forceinline__ void store_state4<float>(float* state,
float4 value) {
*reinterpret_cast<float4*>(state) = value;
}
template <>
__device__ __forceinline__ void store_state4<__nv_bfloat16>(
__nv_bfloat16* state, float4 value) {
*reinterpret_cast<__nv_bfloat162*>(state) =
__floats2bfloat162_rn(value.x, value.y);
*reinterpret_cast<__nv_bfloat162*>(state + 2) =
__floats2bfloat162_rn(value.z, value.w);
}
constexpr int kDimK = 128;
constexpr int kDimV = 128;
constexpr int kThreads = 256;
constexpr int kWarps = kThreads / 32;
constexpr int kChunkV = 32;
constexpr int kNumChunks = kDimV / kChunkV;
constexpr int kRowsPerWarp = kChunkV / kWarps;
constexpr int kMaxMtpTokens = 8;
constexpr int kDtBiasFloat32 = 0;
constexpr int kDtBiasBFloat16 = 1;
constexpr int kDtBiasFloat16 = 2;
struct GdnDecodeStrides {
int64_t mixed_row;
int64_t a_row;
int64_t b_row;
int64_t gate_row;
int64_t state_slot;
};
__device__ __forceinline__ float sigmoid_fast(float x) {
return 1.0f / (1.0f + __expf(-x));
}
__device__ __forceinline__ float silu_fast(float x) {
return x * sigmoid_fast(x);
}
__device__ __forceinline__ float softplus_fast(float x) {
return x > 20.0f ? x : log1pf(__expf(x));
}
__device__ __forceinline__ float load_dt_bias(const void* dt_bias, int head,
int dt_bias_type) {
if (dt_bias_type == kDtBiasBFloat16) {
return __bfloat162float(static_cast<const __nv_bfloat16*>(dt_bias)[head]);
}
if (dt_bias_type == kDtBiasFloat16) {
return __half2float(static_cast<const __half*>(dt_bias)[head]);
}
return static_cast<const float*>(dt_bias)[head];
}
__device__ __forceinline__ float warp_reduce_sum(float value) {
#pragma unroll
for (int offset = 16; offset > 0; offset >>= 1) {
value += __shfl_xor_sync(0xffffffffu, value, offset);
}
return value;
}
struct Sum2 {
float x;
float y;
};
__device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) {
#pragma unroll
for (int offset = 16; offset > 0; offset >>= 1) {
x += __shfl_xor_sync(0xffffffffu, x, offset);
y += __shfl_xor_sync(0xffffffffu, y, offset);
}
return {x, y};
}
template <typename StateT>
__global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
const __nv_bfloat16* __restrict__ mixed_qkv,
const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b,
const float* __restrict__ a_log, const void* __restrict__ dt_bias,
const int* __restrict__ state_indices, const int* __restrict__ cu_seqlens,
const int* __restrict__ num_accepted_tokens, StateT* __restrict__ state,
const __nv_bfloat16* __restrict__ output_gate,
const void* __restrict__ norm_weight, __nv_bfloat16* __restrict__ out,
int H, int HV, int state_indices_width, int dt_bias_type,
bool norm_weight_is_bf16, float scale, float norm_eps,
GdnDecodeStrides strides) {
const int request = blockIdx.x;
const int value_head = blockIdx.y;
const int tid = threadIdx.x;
const int lane = tid & 31;
const int warp = tid >> 5;
const int bos = cu_seqlens[request];
const int eos = cu_seqlens[request + 1];
const int num_tokens = eos - bos;
if (num_tokens <= 0) {
return;
}
const int accepted = num_accepted_tokens[request];
const int source_slot =
accepted > 0 && accepted <= state_indices_width
? state_indices[request * state_indices_width + accepted - 1]
: 0;
if (source_slot <= 0 || num_tokens > kMaxMtpTokens) {
for (int linear = tid; linear < num_tokens * kDimV; linear += kThreads) {
const int token = bos + linear / kDimV;
const int value = linear % kDimV;
const int64_t out_offset =
(static_cast<int64_t>(token) * HV + value_head) * kDimV + value;
out[out_offset] = __float2bfloat16(0.0f);
}
return;
}
const int key_head = value_head / 8;
__shared__ StateT shared_state[2][kChunkV][kDimK];
__shared__ float shared_q[kMaxMtpTokens][kDimK];
__shared__ float shared_k[kMaxMtpTokens][kDimK];
__shared__ __nv_bfloat16 shared_v[kMaxMtpTokens][kDimV];
__shared__ __nv_bfloat16 shared_out[kMaxMtpTokens][kDimV];
__shared__ float shared_decay[kMaxMtpTokens];
__shared__ float shared_beta[kMaxMtpTokens];
StateT* source_state =
state + static_cast<int64_t>(source_slot) * strides.state_slot +
value_head * kDimV * kDimK;
copy_state_chunk<StateT, kChunkV, kDimK, 2>(&shared_state[0][0][0],
source_state, 0, tid, kThreads);
if (warp < num_tokens) {
const int t = warp;
const int token = bos + t;
const int64_t mixed_base = static_cast<int64_t>(token) * strides.mixed_row;
float q_values[4];
float k_values[4];
float q_square = 0.0f;
float k_square = 0.0f;
#pragma unroll
for (int i = 0; i < 4; ++i) {
const int dim = lane + i * 32;
q_values[i] =
__bfloat162float(mixed_qkv[mixed_base + key_head * kDimK + dim]);
k_values[i] = __bfloat162float(
mixed_qkv[mixed_base + H * kDimK + key_head * kDimK + dim]);
shared_v[t][dim] =
mixed_qkv[mixed_base + 2 * H * kDimK + value_head * kDimV + dim];
q_square += q_values[i] * q_values[i];
k_square += k_values[i] * k_values[i];
}
const Sum2 qk_sums = warp_reduce_sum_pair(q_square, k_square);
const float q_scale = __shfl_sync(
0xffffffffu, lane == 0 ? rsqrtf(qk_sums.x + 1.0e-6f) * scale : 0.0f, 0);
const float k_scale = __shfl_sync(
0xffffffffu, lane == 0 ? rsqrtf(qk_sums.y + 1.0e-6f) : 0.0f, 0);
#pragma unroll
for (int i = 0; i < 4; ++i) {
const int dim = lane + i * 32;
shared_q[t][dim] = q_values[i] * q_scale;
shared_k[t][dim] = k_values[i] * k_scale;
}
if (lane == 0) {
const float a_value = __bfloat162float(
a[static_cast<int64_t>(token) * strides.a_row + value_head]);
const float b_value = __bfloat162float(
b[static_cast<int64_t>(token) * strides.b_row + value_head]);
const float g = -__expf(a_log[value_head]) *
softplus_fast(a_value + load_dt_bias(dt_bias, value_head,
dt_bias_type));
shared_decay[t] = __expf(g);
shared_beta[t] = sigmoid_fast(b_value);
}
}
__syncthreads();
const int k_base = lane * 4;
int rows[kRowsPerWarp];
#pragma unroll
for (int row = 0; row < kRowsPerWarp; ++row) {
rows[row] = warp + row * kWarps;
}
#pragma unroll
for (int chunk = 0; chunk < kNumChunks; ++chunk) {
cp_async_wait_all();
__syncthreads();
if (chunk + 1 < kNumChunks) {
copy_state_chunk<StateT, kChunkV, kDimK, 2>(
&shared_state[0][0][0], source_state, chunk + 1, tid, kThreads);
}
float h[kRowsPerWarp][4];
#pragma unroll
for (int row = 0; row < kRowsPerWarp; ++row) {
const float4 state_value =
load_state4(&shared_state[chunk & 1][rows[row]][k_base]);
h[row][0] = state_value.x;
h[row][1] = state_value.y;
h[row][2] = state_value.z;
h[row][3] = state_value.w;
}
for (int t = 0; t < num_tokens; ++t) {
const float4 q4 = *reinterpret_cast<const float4*>(&shared_q[t][k_base]);
const float4 k4 = *reinterpret_cast<const float4*>(&shared_k[t][k_base]);
const float q_values[4] = {q4.x, q4.y, q4.z, q4.w};
const float k_values[4] = {k4.x, k4.y, k4.z, k4.w};
float dot_hk[kRowsPerWarp] = {0.0f, 0.0f, 0.0f, 0.0f};
#pragma unroll
for (int row = 0; row < kRowsPerWarp; ++row) {
#pragma unroll
for (int i = 0; i < 4; ++i) {
h[row][i] *= shared_decay[t];
dot_hk[row] += h[row][i] * k_values[i];
}
}
const Sum2 dot_hk_01 = warp_reduce_sum_pair(dot_hk[0], dot_hk[1]);
const Sum2 dot_hk_23 = warp_reduce_sum_pair(dot_hk[2], dot_hk[3]);
const float reduced_hk[kRowsPerWarp] = {dot_hk_01.x, dot_hk_01.y,
dot_hk_23.x, dot_hk_23.y};
float dot_hq[kRowsPerWarp] = {0.0f, 0.0f, 0.0f, 0.0f};
#pragma unroll
for (int row = 0; row < kRowsPerWarp; ++row) {
const int value = chunk * kChunkV + rows[row];
const float delta =
(__bfloat162float(shared_v[t][value]) - reduced_hk[row]) *
shared_beta[t];
#pragma unroll
for (int i = 0; i < 4; ++i) {
h[row][i] += k_values[i] * delta;
dot_hq[row] += h[row][i] * q_values[i];
}
}
const Sum2 dot_hq_01 = warp_reduce_sum_pair(dot_hq[0], dot_hq[1]);
const Sum2 dot_hq_23 = warp_reduce_sum_pair(dot_hq[2], dot_hq[3]);
if (lane == 0) {
shared_out[t][chunk * kChunkV + rows[0]] =
__float2bfloat16(dot_hq_01.x);
shared_out[t][chunk * kChunkV + rows[1]] =
__float2bfloat16(dot_hq_01.y);
shared_out[t][chunk * kChunkV + rows[2]] =
__float2bfloat16(dot_hq_23.x);
shared_out[t][chunk * kChunkV + rows[3]] =
__float2bfloat16(dot_hq_23.y);
}
const int destination_slot =
state_indices[request * state_indices_width + t];
if (destination_slot > 0) {
StateT* destination_state =
state +
static_cast<int64_t>(destination_slot) * strides.state_slot +
value_head * kDimV * kDimK;
#pragma unroll
for (int row = 0; row < kRowsPerWarp; ++row) {
const int value = chunk * kChunkV + rows[row];
store_state4(destination_state + value * kDimK + k_base,
make_float4(h[row][0], h[row][1], h[row][2], h[row][3]));
}
}
}
}
__syncthreads();
if (warp < num_tokens) {
const int t = warp;
float output_values[4];
float sum_square = 0.0f;
#pragma unroll
for (int i = 0; i < 4; ++i) {
const int value = lane + i * 32;
output_values[i] = __bfloat162float(shared_out[t][value]);
sum_square += output_values[i] * output_values[i];
}
sum_square = warp_reduce_sum(sum_square);
const float rstd =
rsqrtf(sum_square / static_cast<float>(kDimV) + norm_eps);
const int token = bos + t;
#pragma unroll
for (int i = 0; i < 4; ++i) {
const int value = lane + i * 32;
const float gate = silu_fast(__bfloat162float(
output_gate[static_cast<int64_t>(token) * strides.gate_row +
value_head * kDimV + value]));
const float weight =
norm_weight_is_bf16
? __bfloat162float(
static_cast<const __nv_bfloat16*>(norm_weight)[value])
: static_cast<const float*>(norm_weight)[value];
const int64_t out_offset =
(static_cast<int64_t>(token) * HV + value_head) * kDimV + value;
out[out_offset] =
__float2bfloat16(output_values[i] * rstd * weight * gate);
}
}
}
template <typename StateT>
void launch_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log,
torch::stable::Tensor const& dt_bias,
torch::stable::Tensor const& state_indices,
torch::stable::Tensor const& cu_seqlens,
torch::stable::Tensor const& num_accepted_tokens,
torch::stable::Tensor& state, torch::stable::Tensor const& norm_weight,
torch::stable::Tensor& out, const __nv_bfloat16* a, const __nv_bfloat16* b,
const __nv_bfloat16* output_gate, int num_key_heads, int num_value_heads,
double scale, double norm_eps, GdnDecodeStrides strides) {
using torch::headeronly::ScalarType;
const auto dt_bias_scalar_type = dt_bias.scalar_type();
const int dt_bias_type =
dt_bias_scalar_type == ScalarType::Float
? kDtBiasFloat32
: (dt_bias_scalar_type == ScalarType::BFloat16 ? kDtBiasBFloat16
: kDtBiasFloat16);
torch::stable::accelerator::DeviceGuard const device_guard(
mixed_qkv.get_device_index());
const cudaStream_t stream =
get_current_cuda_stream(mixed_qkv.get_device_index());
const int num_requests = static_cast<int>(state_indices.size(0));
const dim3 grid(num_requests, num_value_heads);
gdn_decode_post_conv_mtp_kernel<StateT><<<grid, kThreads, 0, stream>>>(
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
static_cast<const int*>(state_indices.data_ptr()),
static_cast<const int*>(cu_seqlens.data_ptr()),
static_cast<const int*>(num_accepted_tokens.data_ptr()),
static_cast<StateT*>(state.data_ptr()), output_gate,
norm_weight.data_ptr(), static_cast<__nv_bfloat16*>(out.data_ptr()),
num_key_heads, num_value_heads, static_cast<int>(state_indices.size(1)),
dt_bias_type, norm_weight.scalar_type() == ScalarType::BFloat16,
static_cast<float>(scale), static_cast<float>(norm_eps), strides);
const cudaError_t error = cudaGetLastError();
STD_TORCH_CHECK(error == cudaSuccess,
"GDN decode MTP post-conv kernel launch failed: ",
cudaGetErrorString(error));
}
} // namespace
void fused_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a,
torch::stable::Tensor const& b, torch::stable::Tensor const& a_log,
torch::stable::Tensor const& dt_bias,
torch::stable::Tensor const& state_indices,
torch::stable::Tensor const& cu_seqlens,
torch::stable::Tensor const& num_accepted_tokens,
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
double scale, double norm_eps) {
using torch::headeronly::ScalarType;
STD_TORCH_CHECK(
mixed_qkv.is_cuda() && mixed_qkv.scalar_type() == ScalarType::BFloat16,
"mixed_qkv must be a CUDA bfloat16 tensor");
STD_TORCH_CHECK(a.is_cuda() && a.scalar_type() == ScalarType::BFloat16,
"a must be a CUDA bfloat16 tensor");
STD_TORCH_CHECK(b.is_cuda() && b.scalar_type() == ScalarType::BFloat16,
"b must be a CUDA bfloat16 tensor");
STD_TORCH_CHECK(a_log.is_cuda() && a_log.scalar_type() == ScalarType::Float,
"A_log must be a CUDA float32 tensor");
const auto dt_bias_scalar_type = dt_bias.scalar_type();
STD_TORCH_CHECK(
dt_bias.is_cuda() && (dt_bias_scalar_type == ScalarType::Float ||
dt_bias_scalar_type == ScalarType::BFloat16 ||
dt_bias_scalar_type == ScalarType::Half),
"dt_bias must be a CUDA float32, bfloat16, or float16 tensor");
STD_TORCH_CHECK(
state_indices.is_cuda() && state_indices.scalar_type() == ScalarType::Int,
"state_indices must be a CUDA int32 tensor");
STD_TORCH_CHECK(
cu_seqlens.is_cuda() && cu_seqlens.scalar_type() == ScalarType::Int,
"cu_seqlens must be a CUDA int32 tensor");
STD_TORCH_CHECK(num_accepted_tokens.is_cuda() &&
num_accepted_tokens.scalar_type() == ScalarType::Int,
"num_accepted_tokens must be a CUDA int32 tensor");
const auto state_scalar_type = state.scalar_type();
STD_TORCH_CHECK(
state.is_cuda() && (state_scalar_type == ScalarType::Float ||
state_scalar_type == ScalarType::BFloat16),
"state must be a CUDA float32 or bfloat16 tensor");
STD_TORCH_CHECK(output_gate.is_cuda() &&
output_gate.scalar_type() == ScalarType::BFloat16,
"output_gate must be a CUDA bfloat16 tensor");
STD_TORCH_CHECK(norm_weight.is_cuda() &&
(norm_weight.scalar_type() == ScalarType::Float ||
norm_weight.scalar_type() == ScalarType::BFloat16),
"norm_weight must be a CUDA float32 or bfloat16 tensor");
STD_TORCH_CHECK(out.is_cuda() && out.scalar_type() == ScalarType::BFloat16,
"out must be a CUDA bfloat16 tensor");
STD_TORCH_CHECK(mixed_qkv.dim() == 2,
"mixed_qkv must have shape [L, 2 * H * 128 + HV * 128]");
const int num_tokens = static_cast<int>(mixed_qkv.size(0));
STD_TORCH_CHECK(num_tokens > 0,
"GDN decode MTP fusion requires at least one token");
STD_TORCH_CHECK(
state.dim() == 4 && state.size(2) == kDimV && state.size(3) == kDimK,
"state must have shape [slots, HV, 128, 128]");
const int num_value_heads = static_cast<int>(state.size(1));
const int64_t key_width =
mixed_qkv.size(1) - static_cast<int64_t>(num_value_heads) * kDimV;
STD_TORCH_CHECK(key_width > 0 && key_width % (2 * kDimK) == 0,
"mixed_qkv width is inconsistent with state");
const int num_key_heads = static_cast<int>(key_width / (2 * kDimK));
STD_TORCH_CHECK(num_value_heads == 8 * num_key_heads,
"GDN decode MTP fusion requires HV/H=8");
STD_TORCH_CHECK(state_indices.dim() == 2 && state_indices.size(0) > 0 &&
state_indices.size(1) > 0 &&
state_indices.size(1) <= kMaxMtpTokens,
"state_indices must have shape [N, S] with 1 <= S <= 8");
const int num_requests = static_cast<int>(state_indices.size(0));
STD_TORCH_CHECK(
cu_seqlens.dim() == 1 && cu_seqlens.numel() == num_requests + 1,
"cu_seqlens must have N + 1 elements");
STD_TORCH_CHECK(num_accepted_tokens.dim() == 1 &&
num_accepted_tokens.numel() == num_requests,
"num_accepted_tokens must have N elements");
STD_TORCH_CHECK(
a.dim() == 2 && a.size(0) == num_tokens && a.size(1) == num_value_heads,
"a must have shape [L, HV]");
STD_TORCH_CHECK(
b.dim() == 2 && b.size(0) == num_tokens && b.size(1) == num_value_heads,
"b must have shape [L, HV]");
STD_TORCH_CHECK(a_log.is_contiguous() && a_log.numel() == num_value_heads,
"A_log must be contiguous with HV elements");
STD_TORCH_CHECK(dt_bias.is_contiguous() && dt_bias.numel() == num_value_heads,
"dt_bias must be contiguous with HV elements");
STD_TORCH_CHECK(state_indices.is_contiguous(),
"state_indices must be contiguous");
STD_TORCH_CHECK(cu_seqlens.is_contiguous(), "cu_seqlens must be contiguous");
STD_TORCH_CHECK(num_accepted_tokens.is_contiguous(),
"num_accepted_tokens must be contiguous");
STD_TORCH_CHECK(output_gate.dim() == 3 && output_gate.size(0) == num_tokens &&
output_gate.size(1) == num_value_heads &&
output_gate.size(2) == kDimV,
"output_gate must have shape [L, HV, 128]");
STD_TORCH_CHECK(norm_weight.is_contiguous() && norm_weight.numel() == kDimV,
"norm_weight must be contiguous with 128 elements");
STD_TORCH_CHECK(out.dim() == 3 && out.size(0) == num_tokens &&
out.size(1) == num_value_heads && out.size(2) == kDimV,
"out must have shape [L, HV, 128]");
STD_TORCH_CHECK(mixed_qkv.stride(1) == 1,
"mixed_qkv channels must be contiguous");
STD_TORCH_CHECK(a.stride(1) == 1 && b.stride(1) == 1,
"a and b heads must be contiguous");
STD_TORCH_CHECK(state.stride(0) >= num_value_heads * kDimV * kDimK &&
state.stride(1) == kDimV * kDimK &&
state.stride(2) == kDimK && state.stride(3) == 1,
"state must have contiguous [HV, 128, 128] slot contents");
const int state_elements_per_copy =
state_scalar_type == ScalarType::Float ? 4 : 8;
STD_TORCH_CHECK(reinterpret_cast<uintptr_t>(state.data_ptr()) % 16 == 0 &&
state.stride(0) % state_elements_per_copy == 0,
"state slots must preserve 16-byte alignment");
STD_TORCH_CHECK(output_gate.stride(2) == 1 && output_gate.stride(1) == kDimV,
"output_gate head rows must be contiguous");
STD_TORCH_CHECK(out.is_contiguous(), "out must be contiguous");
STD_TORCH_CHECK(norm_eps >= 0.0, "norm_eps must be non-negative");
const GdnDecodeStrides strides{mixed_qkv.stride(0), a.stride(0), b.stride(0),
output_gate.stride(0), state.stride(0)};
const auto* a_ptr = static_cast<const __nv_bfloat16*>(a.data_ptr());
const auto* b_ptr = static_cast<const __nv_bfloat16*>(b.data_ptr());
const auto* output_gate_ptr =
static_cast<const __nv_bfloat16*>(output_gate.data_ptr());
if (state_scalar_type == ScalarType::Float) {
launch_gdn_decode_post_conv_mtp<float>(
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
strides);
} else {
launch_gdn_decode_post_conv_mtp<__nv_bfloat16>(
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
strides);
}
}
+12
View File
@@ -400,6 +400,18 @@ void fused_kda_decode(
torch::stable::Tensor& out, std::optional<double> lower_bound,
std::optional<torch::stable::Tensor> output_gate,
std::optional<torch::stable::Tensor> norm_weight, double norm_eps);
void fused_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a,
torch::stable::Tensor const& b, torch::stable::Tensor const& a_log,
torch::stable::Tensor const& dt_bias,
torch::stable::Tensor const& state_indices,
torch::stable::Tensor const& cu_seqlens,
torch::stable::Tensor const& num_accepted_tokens,
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
double scale, double norm_eps);
#endif
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
+14
View File
@@ -534,6 +534,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"Tensor? norm_weight=None, float norm_eps=1e-5) -> ()");
#endif
#ifdef VLLM_ENABLE_FUSED_GDN_DECODE
ops.def(
"fused_gdn_decode_post_conv_mtp("
"Tensor mixed_qkv, Tensor a, Tensor b, Tensor A_log, Tensor dt_bias, "
"Tensor state_indices, Tensor cu_seqlens, Tensor num_accepted_tokens, "
"Tensor! state, Tensor output_gate, Tensor norm_weight, Tensor! out, "
"float scale, float norm_eps=1e-5) -> ()");
#endif
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
ops.def(
"kimi_k3_attn_res("
@@ -797,6 +806,11 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
ops.impl("fused_kda_decode", TORCH_BOX(&fused_kda_decode));
#endif
#ifdef VLLM_ENABLE_FUSED_GDN_DECODE
ops.impl("fused_gdn_decode_post_conv_mtp",
TORCH_BOX(&fused_gdn_decode_post_conv_mtp));
#endif
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res));
#endif
+5
View File
@@ -252,6 +252,11 @@ def test_splitting_ops_dynamic():
# populated when the engine decides to use piecewise compilation.
assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
assert config.compilation_config.splitting_ops_contain_attention()
splitting_ops = config.compilation_config.splitting_ops
assert splitting_ops is not None
assert {
"vllm::qwen_gdn_attention_core_fused_norm_packed",
} <= set(splitting_ops)
# When use_inductor_graph_partition=True
config = VllmConfig(
+345
View File
@@ -0,0 +1,345 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model-path tests for fused Qwen3.5 GDN MTP decode."""
from __future__ import annotations
import types
from unittest.mock import patch
import pytest
import torch
from vllm.platforms import current_platform
if not (current_platform.is_cuda() and current_platform.has_device_capability(80)):
pytest.skip(
reason="Fused GDN MTP decode requires CUDA compute capability 8.0+.",
allow_module_level=True,
)
from tests.v1.attention.utils import ( # noqa: E402
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig, set_current_vllm_config # noqa: E402
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn # noqa: E402
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( # noqa: E402
ChunkGatedDeltaRule,
QwenGatedDeltaNetAttention,
)
from vllm.model_executor.layers.mamba.mamba_utils import ( # noqa: E402
MambaStateShapeCalculator,
)
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( # noqa: E402
rmsnorm_fn,
)
from vllm.utils.torch_utils import _encode_layer_name # noqa: E402
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
GDNAttentionMetadataBuilder,
)
from vllm.v1.kv_cache_interface import MambaSpec # noqa: E402
NUM_SPEC = 3
SPEC_TOKENS = NUM_SPEC + 1
H = 1
HV = 8
K = 128
V = 128
CONV_KERNEL = 4
CONV_DIM = 2 * H * K + HV * V
BLOCK_SIZE = 16
PREFIX = "model.layers.0.linear_attn"
EPS = 1e-6
class _TestGatedNorm:
def __init__(self, weight: torch.Tensor) -> None:
self.weight = weight
self.bias = None
self.eps = EPS
self.group_size = None
self.norm_before_gate = True
self.activation = "silu"
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
return rmsnorm_fn(
x,
self.weight,
None,
z=z,
eps=self.eps,
norm_before_gate=True,
activation=self.activation,
)
def _make_vllm_config():
config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
hf_config_override={"linear_key_head_dim": K},
)
config.additional_config = {"gdn_prefill_backend": "cutedsl"}
config.cache_config.mamba_cache_mode = "none"
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=NUM_SPEC
)
return config
def _build_layer(
vllm_config,
conv_state: torch.Tensor,
ssm_state: torch.Tensor,
a_log: torch.Tensor,
dt_bias: torch.Tensor,
conv_weight: torch.Tensor,
norm_weight: torch.Tensor,
):
layer = types.SimpleNamespace(
prefix=PREFIX,
enable_packed_recurrent_decode=False,
disable_tp_for_ba_proj=False,
tp_size=1,
num_k_heads=H,
num_v_heads=HV,
head_k_dim=K,
head_v_dim=V,
key_dim=K,
value_dim=HV * V,
activation="silu",
A_log=a_log,
dt_bias=dt_bias,
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
kv_cache=(conv_state, ssm_state),
norm=_TestGatedNorm(norm_weight),
layer_norm_epsilon=EPS,
gdn_decode_kernel="cuda",
)
with set_current_vllm_config(vllm_config):
layer.chunk_gated_delta_rule = ChunkGatedDeltaRule()
for name in (
"rearrange_mixed_qkv",
"_forward_core",
"_forward_core_decode_spec_post_conv_fused_norm",
"_forward_core_decode_spec_fused_norm",
"_can_use_fused_gdn_mtp_decode",
"_rms_norm_gated_cuda",
"_forward_core_fused_norm",
"_forward_core_fused_norm_packed",
"split_ba",
):
setattr(
layer,
name,
types.MethodType(getattr(QwenGatedDeltaNetAttention, name), layer),
)
return layer
@torch.inference_mode()
def test_fused_forward_uses_packed_entrypoint() -> None:
"""Fused mode keeps projected QKVZ and BA packed through the model op."""
device = torch.device("cuda")
num_tokens = 3
hidden_states = torch.empty(num_tokens, 1, dtype=torch.bfloat16, device=device)
mixed_qkvz = torch.randn(
num_tokens, CONV_DIM + HV * V, dtype=torch.bfloat16, device=device
)
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
layer = types.SimpleNamespace(
prefix=PREFIX,
enable_fused_gdn_decode=True,
norm=types.SimpleNamespace(
weight=torch.empty(V, dtype=torch.bfloat16, device=device)
),
num_v_heads=HV,
tp_size=1,
head_v_dim=V,
in_proj_qkvz=lambda _: (mixed_qkvz, None),
in_proj_ba=lambda _: (ba, None),
out_proj=lambda x: (x, None),
)
layer.forward_cuda = types.MethodType(
QwenGatedDeltaNetAttention.forward_cuda, layer
)
def packed_op(
actual_qkvz: torch.Tensor,
actual_ba: torch.Tensor,
output: torch.Tensor,
*,
layer_name: str,
) -> None:
assert actual_qkvz is mixed_qkvz
assert actual_ba is ba
assert layer_name == _encode_layer_name(PREFIX)
output.fill_(1)
with (
patch.object(
torch.ops.vllm,
"qwen_gdn_attention_core_fused_norm_packed",
side_effect=packed_op,
) as packed_mock,
patch.object(torch.ops.vllm, "qwen_gdn_attention_core") as triton_mock,
):
output = layer.forward_cuda(hidden_states)
packed_mock.assert_called_once()
triton_mock.assert_not_called()
torch.testing.assert_close(output, torch.ones_like(output))
@pytest.mark.parametrize(
"seq_lens,query_lens,draft_tokens,expected_fused_calls",
[
pytest.param([128], [SPEC_TOKENS], [NUM_SPEC], 1, id="pure-mtp"),
pytest.param(
[128, 96],
[SPEC_TOKENS, 64],
[NUM_SPEC, -1],
0,
id="mixed-mtp-falls-back",
),
pytest.param([96], [64], [-1], 0, id="pure-prefill"),
pytest.param([128], [1], [-1], 0, id="pure-decode"),
],
)
@torch.inference_mode()
def test_fused_model_path_matches_reference(
seq_lens: list[int],
query_lens: list[int],
draft_tokens: list[int],
expected_fused_calls: int,
) -> None:
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
torch.manual_seed(1)
device = torch.device("cuda")
vllm_config = _make_vllm_config()
builder = GDNAttentionMetadataBuilder(
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=NUM_SPEC,
),
layer_names=[PREFIX],
vllm_config=vllm_config,
device=device,
)
batch = BatchSpec(seq_lens=seq_lens, query_lens=query_lens)
common = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
)
common.block_table_tensor.add_(1)
with set_current_vllm_config(vllm_config):
metadata = builder.build(
common_prefix_len=0,
common_attn_metadata=common,
num_accepted_tokens=torch.ones(
batch.batch_size, dtype=torch.int32, device=device
),
num_decode_draft_tokens_cpu=torch.tensor(draft_tokens, dtype=torch.int32),
)
state_indices = [
indices
for indices in (
metadata.spec_state_indices_tensor,
metadata.non_spec_state_indices_tensor,
)
if indices is not None and indices.numel() > 0
]
pool_size = max(int(indices.max().item()) for indices in state_indices) + 1
conv_state_shape, temporal_state_shape = (
MambaStateShapeCalculator.gated_delta_net_state_shape(
1, H, HV, K, V, CONV_KERNEL, NUM_SPEC
)
)
conv_state_seed = 0.05 * torch.randn(
pool_size, *conv_state_shape, dtype=torch.bfloat16, device=device
)
ssm_state_seed = 0.01 * torch.randn(
pool_size, *temporal_state_shape, dtype=torch.float32, device=device
)
a_log = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
conv_weight = 0.1 * torch.randn(
CONV_DIM, 1, CONV_KERNEL, dtype=torch.bfloat16, device=device
)
norm_weight = torch.randn(V, dtype=torch.float32, device=device)
num_tokens = batch.compute_num_tokens()
mixed_qkv = 0.1 * torch.randn(
num_tokens, CONV_DIM, dtype=torch.bfloat16, device=device
)
b = 0.1 * torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device)
a = 0.1 * torch.randn_like(b)
output_gate = 0.1 * torch.randn(
num_tokens, HV, V, dtype=torch.bfloat16, device=device
)
mixed_qkvz = torch.cat((mixed_qkv, output_gate.flatten(1)), dim=-1)
ba = torch.cat((b, a), dim=-1)
context = types.SimpleNamespace(attn_metadata={PREFIX: metadata})
reference_layer = _build_layer(
vllm_config,
conv_state_seed.clone(),
ssm_state_seed.clone(),
a_log,
dt_bias,
conv_weight,
norm_weight,
)
reference_out = torch.zeros_like(output_gate)
with patch.object(
qwen_gdn_linear_attn, "get_forward_context", return_value=context
):
reference_layer._forward_core(
mixed_qkv=mixed_qkv.clone(),
b=b,
a=a,
core_attn_out=reference_out,
)
reference_out = reference_layer.norm(reference_out, output_gate)
fused_layer = _build_layer(
vllm_config,
conv_state_seed.clone(),
ssm_state_seed.clone(),
a_log,
dt_bias,
conv_weight,
norm_weight,
)
context.no_compile_layers = {PREFIX: fused_layer}
fused_out = torch.zeros_like(output_gate)
fused_op = qwen_gdn_linear_attn.ops.fused_gdn_decode_post_conv_mtp
with (
patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=context),
patch.object(
qwen_gdn_linear_attn.ops,
"fused_gdn_decode_post_conv_mtp",
wraps=fused_op,
) as fused_mock,
):
torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed(
mixed_qkvz.clone(),
ba,
fused_out,
layer_name=_encode_layer_name(PREFIX),
)
assert fused_mock.call_count == expected_fused_calls
torch.testing.assert_close(
fused_layer.kv_cache[0], reference_layer.kv_cache[0], atol=0, rtol=0
)
torch.testing.assert_close(fused_out, reference_out, atol=3e-2, rtol=3e-2)
torch.testing.assert_close(
fused_layer.kv_cache[1],
reference_layer.kv_cache[1],
atol=3e-2,
rtol=3e-2,
)
+139 -3
View File
@@ -1,18 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for fused_gdn_prefill_post_conv kernel.
"""Tests for GDN post-convolution kernels.
Verifies that the fused kernel matches the reference:
split → rearrange → contiguous → l2norm → gating
The prefill tests cover preparation and the MTP tests cover recurrent state
updates plus output normalization and gating.
"""
import pytest
import torch
import torch.nn.functional as F
from vllm import _custom_ops as ops
from vllm.third_party.flash_linear_attention.ops import (
fused_sigmoid_gating_delta_rule_update,
)
from vllm.third_party.flash_linear_attention.ops.fused_gdn_prefill_post_conv import (
fused_post_conv_prep,
)
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
rmsnorm_fn,
)
def reference_post_conv(
@@ -207,3 +214,132 @@ def test_fused_post_conv_l0():
)
assert q.shape == (0, H, K)
assert g.shape == (0, HV)
@pytest.mark.parametrize(
"tp_size,query_lengths,state_dtype,norm_dtype",
[
pytest.param(16, (4, 4), torch.bfloat16, torch.bfloat16, id="tp16-bf16"),
pytest.param(4, (4, 4), torch.float32, torch.float32, id="tp4-fp32"),
pytest.param(16, (4, 2, 0), torch.bfloat16, torch.float32, id="tp16-ragged"),
pytest.param(4, (4, 2, 0), torch.float32, torch.bfloat16, id="tp4-ragged"),
pytest.param(16, (8,), torch.float32, torch.bfloat16, id="tp16-max"),
pytest.param(4, (8,), torch.bfloat16, torch.float32, id="tp4-max"),
],
)
@torch.inference_mode()
def test_fused_gdn_decode_post_conv_mtp_ratio8(
tp_size: int,
query_lengths: tuple[int, ...],
state_dtype: torch.dtype,
norm_dtype: torch.dtype,
) -> None:
if torch.cuda.get_device_capability() < (8, 0):
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
pytest.skip("fused GDN decode MTP op is not built")
torch.manual_seed(0)
device = "cuda"
H = 16 // tp_size
HV = 128 // tp_size
K = V = 128
num_reqs = len(query_lengths)
state_width = max(query_lengths)
num_tokens = sum(query_lengths)
num_slots = num_reqs * state_width + 1
scale = K**-0.5
eps = 1e-6
mixed_qkv = torch.randn(
num_tokens,
2 * H * K + HV * V,
dtype=torch.bfloat16,
device=device,
)
query, key, value = torch.split(
mixed_qkv,
[H * K, H * K, HV * V],
dim=-1,
)
query = query.view(1, num_tokens, H, K)
key = key.view(1, num_tokens, H, K)
value = value.view(1, num_tokens, HV, V)
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
b, a = ba.chunk(2, dim=-1)
assert not a.is_contiguous()
assert not b.is_contiguous()
A_log = 0.5 * torch.randn(HV, dtype=torch.float32, device=device)
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
output_gate = torch.randn(num_tokens, HV, V, dtype=torch.bfloat16, device=device)
norm_weight = torch.randn(V, dtype=norm_dtype, device=device)
state_ref = (
0.01 * torch.randn(num_slots, HV, V, K, dtype=torch.float32, device=device)
).to(state_dtype)
state_actual = state_ref.clone()
state_indices = torch.arange(1, num_slots, dtype=torch.int32, device=device).view(
num_reqs, state_width
)
cu_seqlens = torch.tensor(
[0, *torch.tensor(query_lengths).cumsum(0).tolist()],
dtype=torch.int32,
device=device,
)
num_accepted_tokens = torch.ones(num_reqs, dtype=torch.int32, device=device)
if query_lengths[-1] == 0:
state_indices[-1].zero_()
for step, accepted_tokens in enumerate((1, min(2, state_width), state_width)):
num_accepted_tokens.fill_(accepted_tokens)
if query_lengths[-1] == 0:
num_accepted_tokens[-1] = 1
raw_ref, _ = fused_sigmoid_gating_delta_rule_update(
A_log=A_log,
a=a,
b=b,
dt_bias=dt_bias,
q=query,
k=key,
v=value,
initial_state=state_ref,
inplace_final_state=True,
cu_seqlens=cu_seqlens,
ssm_state_indices=state_indices,
num_accepted_tokens=num_accepted_tokens,
scale=scale,
use_qk_l2norm_in_kernel=True,
)
expected = rmsnorm_fn(
raw_ref.squeeze(0),
norm_weight,
None,
z=output_gate,
eps=eps,
norm_before_gate=True,
activation="silu",
)
actual = ops.fused_gdn_decode_post_conv_mtp(
mixed_qkv=mixed_qkv,
a=a,
b=b,
A_log=A_log,
dt_bias=dt_bias,
state_indices=state_indices,
cu_seqlens=cu_seqlens,
num_accepted_tokens=num_accepted_tokens,
state=state_actual,
output_gate=output_gate,
norm_weight=norm_weight,
out=torch.empty_like(output_gate),
scale=scale,
norm_eps=eps,
)
output_error = (actual.float() - expected.float()).norm()
output_relative_l2 = output_error / expected.float().norm().clamp_min(1e-20)
assert output_relative_l2 < 5e-4, (
f"MTP output relative L2 mismatch at step {step}: "
f"{output_relative_l2.item():.6g}"
)
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
+15
View File
@@ -251,6 +251,21 @@ class TestEnvWithChoices:
env_func()
def test_gdn_decode_kernel_env(monkeypatch: pytest.MonkeyPatch):
env_func = environment_variables["VLLM_GDN_DECODE_KERNEL"]
monkeypatch.delenv("VLLM_GDN_DECODE_KERNEL", raising=False)
assert env_func() == "cuda"
for value in ("cuda", "triton"):
monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value)
assert env_func() == value
for value in ("fused", "invalid"):
monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value)
with pytest.raises(ValueError, match="VLLM_GDN_DECODE_KERNEL"):
env_func()
class TestEnvListWithChoices:
"""Test cases for env_list_with_choices function."""
+37
View File
@@ -2777,6 +2777,43 @@ def fused_kda_decode(
return out
def fused_gdn_decode_post_conv_mtp(
mixed_qkv: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
state_indices: torch.Tensor,
cu_seqlens: torch.Tensor,
num_accepted_tokens: torch.Tensor,
state: torch.Tensor,
output_gate: torch.Tensor,
norm_weight: torch.Tensor,
out: torch.Tensor | None = None,
scale: float = 128**-0.5,
norm_eps: float = 1e-5,
) -> torch.Tensor:
if out is None:
out = torch.empty_like(output_gate)
torch.ops._C.fused_gdn_decode_post_conv_mtp(
mixed_qkv,
a,
b,
A_log,
dt_bias,
state_indices,
cu_seqlens,
num_accepted_tokens,
state,
output_gate,
norm_weight,
out,
scale,
norm_eps,
)
return out
def concat_and_cache_mla(
kv_c: torch.Tensor,
k_pe: torch.Tensor,
+1
View File
@@ -769,6 +769,7 @@ class CompilationConfig:
"vllm::short_conv",
"vllm::linear_attention",
"vllm::qwen_gdn_attention_core",
"vllm::qwen_gdn_attention_core_fused_norm_packed",
"vllm::gdn_attention_core_xpu",
"vllm::olmo_hybrid_gdn_full_forward",
"vllm::sparse_attn_indexer",
+10
View File
@@ -127,6 +127,7 @@ if TYPE_CHECKING:
VLLM_DISABLED_KERNELS: list[str] = []
VLLM_USE_HW_AGNOSTIC: bool = False
VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True
VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda"
VLLM_DISABLE_PYNCCL: bool = False
VLLM_USE_OINK_OPS: bool = False
VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD: bool = True
@@ -1209,6 +1210,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool(
int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1"))
),
# Select the GDN MTP decode implementation. "cuda" uses the fused decode
# kernel where supported and falls back to "triton" otherwise; setting it
# explicitly to "cuda" raises when unsupported.
"VLLM_GDN_DECODE_KERNEL": env_with_choices(
"VLLM_GDN_DECODE_KERNEL",
"cuda",
["cuda", "triton"],
case_sensitive=False,
),
# Disable pynccl (using torch.distributed instead)
"VLLM_DISABLE_PYNCCL": lambda: (
os.getenv("VLLM_DISABLE_PYNCCL", "False").lower() in ("true", "1")
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen3-Next/Qwen3.5 model."""
import os
from typing import Literal
import torch
from einops import rearrange
from torch import nn
from vllm import _custom_ops as ops
from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import (
@@ -84,6 +86,9 @@ if GDN_AITER_TRITON_AVAILABLE:
logger = init_logger(__name__)
MAX_FUSED_GDN_MTP_TOKENS = 8
FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
def _resolve_gdn_prefill_backend(
vllm_config: VllmConfig,
@@ -484,12 +489,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.enable_packed_recurrent_decode = (
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
)
self.gdn_decode_kernel = envs.VLLM_GDN_DECODE_KERNEL.strip().lower()
if self.gdn_decode_kernel == "cuda":
reason = self._fused_gdn_decode_unsupported_reason(vllm_config)
if reason is not None:
if "VLLM_GDN_DECODE_KERNEL" in os.environ:
raise ValueError(
f"VLLM_GDN_DECODE_KERNEL=cuda is not supported: {reason}"
)
logger.info_once(
"Falling back to the Triton GDN decode path: %s", reason
)
self.gdn_decode_kernel = "triton"
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def _fused_gdn_decode_unsupported_reason(
self, vllm_config: VllmConfig
) -> str | None:
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()
if (
self.gqa_interleaved_layout
or self.head_k_dim != 128
or self.head_v_dim != 128
or self.norm.activation != "silu"
or vllm_config.model_config.dtype != torch.bfloat16
or conv_state_dtype != torch.bfloat16
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
or not current_platform.has_device_capability(80)
):
return (
"the fused CUDA kernel requires a BF16 GDN model with "
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
"convolution cache, BF16 or FP32 recurrent state, and a "
"GPU with compute capability 8.0+"
)
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
return None
def create_qkvz_proj(
self,
hidden_size: int,
@@ -846,6 +889,26 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
ba, _ = self.in_proj_ba(hidden_states)
use_fused_gdn_decode = (
self.enable_fused_gdn_decode
and hidden_states.dtype == torch.bfloat16
and self.norm.weight.dtype in (torch.bfloat16, torch.float32)
)
if use_fused_gdn_decode:
core_attn_out = torch.zeros(
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed(
mixed_qkvz,
ba,
core_attn_out,
layer_name=_encode_layer_name(self.prefix),
)
output, _ = self.out_proj(core_attn_out.flatten(-2))
return output
if self.gqa_interleaved_layout:
# Qwen3-Next: unpack the interleaved GQA layout
query, key, value, z, b, a = self.fix_query_key_value_ordering(
@@ -862,8 +925,6 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
mixed_qkv, z = mixed_qkvz.split([qkv_size, z_size], dim=-1)
z = z.reshape(z.size(0), -1, self.head_v_dim)
b, a = self.split_ba(ba)
b = b.contiguous()
a = a.contiguous()
# ============================================================
# Part 2: Core Attention (Custom Op)
@@ -878,8 +939,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
torch.ops.vllm.qwen_gdn_attention_core(
mixed_qkv,
b,
a,
b.contiguous(),
a.contiguous(),
core_attn_out,
layer_name=_encode_layer_name(self.prefix),
)
@@ -1622,6 +1683,211 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
)
return
def _forward_core_decode_spec_fused_norm(
self,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
output_gate: torch.Tensor,
core_attn_out: torch.Tensor,
attn_metadata: GDNAttentionMetadata,
) -> None:
state_indices = attn_metadata.spec_state_indices_tensor
cu_seqlens = attn_metadata.spec_query_start_loc
num_accepted_tokens = attn_metadata.num_accepted_tokens
assert state_indices is not None
assert cu_seqlens is not None
assert num_accepted_tokens is not None
num_requests = attn_metadata.num_spec_decodes
num_actual_tokens = attn_metadata.num_actual_tokens
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
)
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
mixed_qkv = causal_conv1d_update(
mixed_qkv[:num_actual_tokens],
conv_state,
conv_weights,
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices[:num_requests, 0],
num_accepted_tokens=num_accepted_tokens[:num_requests],
query_start_loc=cu_seqlens[: num_requests + 1],
max_query_len=state_indices.size(1),
validate_data=False,
)
self._forward_core_decode_spec_post_conv_fused_norm(
mixed_qkv=mixed_qkv,
b=b[:num_actual_tokens],
a=a[:num_actual_tokens],
output_gate=output_gate[:num_actual_tokens],
core_attn_out=core_attn_out[:num_actual_tokens],
attn_metadata=attn_metadata,
)
def _forward_core_decode_spec_post_conv_fused_norm(
self,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
output_gate: torch.Tensor,
core_attn_out: torch.Tensor,
attn_metadata: GDNAttentionMetadata,
) -> None:
state_indices = attn_metadata.spec_state_indices_tensor
cu_seqlens = attn_metadata.spec_query_start_loc
num_accepted_tokens = attn_metadata.num_accepted_tokens
assert state_indices is not None
assert cu_seqlens is not None
assert num_accepted_tokens is not None
num_requests = attn_metadata.num_spec_decodes
ops.fused_gdn_decode_post_conv_mtp(
mixed_qkv=mixed_qkv,
a=a,
b=b,
A_log=self.A_log,
dt_bias=self.dt_bias,
state_indices=state_indices[:num_requests],
cu_seqlens=cu_seqlens[: num_requests + 1],
num_accepted_tokens=num_accepted_tokens[:num_requests],
state=self.kv_cache[1],
output_gate=output_gate,
norm_weight=self.norm.weight,
out=core_attn_out,
scale=self.head_k_dim**-0.5,
norm_eps=self.layer_norm_epsilon,
)
def _forward_core_fused_norm_packed(
self,
mixed_qkvz: torch.Tensor,
ba: torch.Tensor,
core_attn_out: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
if attn_metadata_raw is None:
self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0)
return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata)
mixed_qkv, output_gate_flat = mixed_qkvz.split(
[qkv_size, self.value_dim // self.tp_size], dim=-1
)
output_gate = output_gate_flat.reshape(
output_gate_flat.size(0), -1, self.head_v_dim
)
b, a = self.split_ba(ba)
self._forward_core_fused_norm(
mixed_qkv=mixed_qkv,
b=b,
a=a,
output_gate=output_gate,
core_attn_out=core_attn_out,
)
def _can_use_fused_gdn_mtp_decode(
self, attn_metadata: GDNAttentionMetadata
) -> bool:
state_indices = attn_metadata.spec_state_indices_tensor
return (
attn_metadata.spec_sequence_masks is not None
and attn_metadata.num_decodes == 0
and attn_metadata.num_spec_decodes > 0
and self.kv_cache[1].dtype in FUSED_GDN_STATE_DTYPES
and self.gdn_decode_kernel == "cuda"
and self.num_v_heads == 8 * self.num_k_heads
and state_indices is not None
and state_indices.size(1) <= MAX_FUSED_GDN_MTP_TOKENS
and hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp")
)
def _rms_norm_gated_cuda(
self,
x: torch.Tensor,
output_gate: torch.Tensor,
out: torch.Tensor,
) -> None:
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
layer_norm_fwd,
)
x_shape = x.shape
assert output_gate.shape == x_shape
assert out.shape == x_shape
x_2d = x.reshape(-1, x_shape[-1])
output_gate_2d = output_gate.reshape(-1, x_shape[-1])
out_2d = out.reshape(-1, x_shape[-1])
assert x_2d.stride(-1) == 1
assert output_gate_2d.stride(-1) == 1
assert out_2d.stride(-1) == 1
layer_norm_fwd(
x_2d,
self.norm.weight.contiguous(),
self.norm.bias,
self.norm.eps,
z=output_gate_2d,
out=out_2d,
group_size=(
x_shape[-1] if self.norm.group_size is None else self.norm.group_size
),
norm_before_gate=self.norm.norm_before_gate,
is_rms_norm=True,
activation=self.norm.activation,
)
def _forward_core_fused_norm(
self,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
output_gate: torch.Tensor,
core_attn_out: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None:
self._warmup_prefill_kernels(mixed_qkv, 0)
return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata)
if (
self._can_use_fused_gdn_mtp_decode(attn_metadata)
and attn_metadata.num_prefills == 0
):
self._forward_core_decode_spec_fused_norm(
mixed_qkv=mixed_qkv,
b=b,
a=a,
output_gate=output_gate,
core_attn_out=core_attn_out,
attn_metadata=attn_metadata,
)
return
self._forward_core(
mixed_qkv=mixed_qkv,
b=b.contiguous(),
a=a.contiguous(),
core_attn_out=core_attn_out,
)
num_actual_tokens = attn_metadata.num_actual_tokens
self._rms_norm_gated_cuda(
core_attn_out[:num_actual_tokens],
output_gate[:num_actual_tokens],
core_attn_out[:num_actual_tokens],
)
def qwen_gdn_attention_core(
qkv_or_qkvz: torch.Tensor,
@@ -1683,6 +1949,39 @@ direct_register_custom_op(
)
def qwen_gdn_attention_core_fused_norm_packed(
mixed_qkvz: torch.Tensor,
ba: torch.Tensor,
core_attn_out: torch.Tensor,
layer_name: LayerNameType,
) -> None:
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._forward_core_fused_norm_packed(
mixed_qkvz=mixed_qkvz,
ba=ba,
core_attn_out=core_attn_out,
)
def gdn_attention_core_fused_norm_packed_fake(
mixed_qkvz: torch.Tensor,
ba: torch.Tensor,
core_attn_out: torch.Tensor,
layer_name: LayerNameType,
) -> None:
return
direct_register_custom_op(
op_name="qwen_gdn_attention_core_fused_norm_packed",
op_func=qwen_gdn_attention_core_fused_norm_packed,
mutates_args=["core_attn_out"],
fake_impl=gdn_attention_core_fused_norm_packed_fake,
)
@triton.jit
def fused_gdn_gating_kernel(
g,