diff --git a/CMakeLists.txt b/CMakeLists.txt index cdcb41bef6..e261bf1ac8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1118,6 +1118,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) cuda_archs_loose_intersection(FUSED_KDA_DECODE_ARCHS "9.0a;10.0f;12.0f" "${CUDA_ARCHS}") + cuda_archs_loose_intersection(FUSED_GDN_DECODE_ARCHS + "8.0;8.6;8.9;9.0a;10.0f;12.0f" "${CUDA_ARCHS}") endif() if(FUSED_KDA_DECODE_ARCHS) set(FUSED_KDA_DECODE_SRC @@ -1131,6 +1133,18 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") message(STATUS "Building fused KDA decode for archs: ${FUSED_KDA_DECODE_ARCHS}") endif() + if(FUSED_GDN_DECODE_ARCHS) + set(FUSED_GDN_DECODE_SRC + "csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu") + set_gencode_flags_for_srcs( + SRCS "${FUSED_GDN_DECODE_SRC}" + CUDA_ARCHS "${FUSED_GDN_DECODE_ARCHS}") + set_property(SOURCE ${FUSED_GDN_DECODE_SRC} APPEND PROPERTY + COMPILE_OPTIONS "$<$:--use_fast_math>") + list(APPEND VLLM_STABLE_EXT_SRC "${FUSED_GDN_DECODE_SRC}") + message(STATUS + "Building fused GDN decode for archs: ${FUSED_GDN_DECODE_ARCHS}") + endif() if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) cuda_archs_loose_intersection(KIMI_K3_ATTN_RES_ARCHS @@ -1213,6 +1227,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") target_compile_definitions(_C_stable_libtorch PRIVATE VLLM_ENABLE_FUSED_KDA_DECODE=1) endif() + if(FUSED_GDN_DECODE_ARCHS) + target_compile_definitions(_C_stable_libtorch PRIVATE + VLLM_ENABLE_FUSED_GDN_DECODE=1) + endif() if(KIMI_K3_ATTN_RES_ARCHS) target_compile_definitions(_C_stable_libtorch PRIVATE VLLM_ENABLE_KIMI_K3_ATTN_RES=1) diff --git a/csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu b/csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu new file mode 100644 index 0000000000..31a16fcf99 --- /dev/null +++ b/csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu @@ -0,0 +1,557 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Copyright contributors to the vLLM project + */ + +#include +#include +#include +#include + +#include "../torch_utils.h" +#include "../../cuda_compat.h" + +namespace { + +template +__device__ __forceinline__ void cp_async_16b(StateT* smem_ptr, + const StateT* gmem_ptr) { + const uint32_t smem_addr = + static_cast(__cvta_generic_to_shared(smem_ptr)); + asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" + : + : "r"(smem_addr), "l"(gmem_ptr)); +} + +__device__ __forceinline__ void cp_async_commit() { + asm volatile("cp.async.commit_group;\n" ::); +} + +__device__ __forceinline__ void cp_async_wait_all() { + asm volatile("cp.async.wait_all;\n" ::: "memory"); +} + +template +__device__ __forceinline__ void copy_state_chunk(StateT* shared_state, + const StateT* state, int chunk, + int thread, int threads) { + constexpr int kElementsPerCopy = 16 / sizeof(StateT); + constexpr int kCopiesPerChunk = ChunkV * DimK / kElementsPerCopy; + const int stage = chunk % Stages; + for (int copy = thread; copy < kCopiesPerChunk; copy += threads) { + const int element = copy * kElementsPerCopy; + cp_async_16b(shared_state + stage * ChunkV * DimK + element, + state + chunk * ChunkV * DimK + element); + } + cp_async_commit(); +} + +template +__device__ __forceinline__ float4 load_state4(const StateT* state); + +template <> +__device__ __forceinline__ float4 load_state4(const float* state) { + return *reinterpret_cast(state); +} + +template <> +__device__ __forceinline__ float4 +load_state4<__nv_bfloat16>(const __nv_bfloat16* state) { + const __nv_bfloat162 lo = *reinterpret_cast(state); + const __nv_bfloat162 hi = *reinterpret_cast(state + 2); + return make_float4(__bfloat162float(lo.x), __bfloat162float(lo.y), + __bfloat162float(hi.x), __bfloat162float(hi.y)); +} + +template +__device__ __forceinline__ void store_state4(StateT* state, float4 value); + +template <> +__device__ __forceinline__ void store_state4(float* state, + float4 value) { + *reinterpret_cast(state) = value; +} + +template <> +__device__ __forceinline__ void store_state4<__nv_bfloat16>( + __nv_bfloat16* state, float4 value) { + *reinterpret_cast<__nv_bfloat162*>(state) = + __floats2bfloat162_rn(value.x, value.y); + *reinterpret_cast<__nv_bfloat162*>(state + 2) = + __floats2bfloat162_rn(value.z, value.w); +} + +constexpr int kDimK = 128; +constexpr int kDimV = 128; +constexpr int kThreads = 256; +constexpr int kWarps = kThreads / 32; +constexpr int kChunkV = 32; +constexpr int kNumChunks = kDimV / kChunkV; +constexpr int kRowsPerWarp = kChunkV / kWarps; +constexpr int kMaxMtpTokens = 8; +constexpr int kDtBiasFloat32 = 0; +constexpr int kDtBiasBFloat16 = 1; +constexpr int kDtBiasFloat16 = 2; + +struct GdnDecodeStrides { + int64_t mixed_row; + int64_t a_row; + int64_t b_row; + int64_t gate_row; + int64_t state_slot; +}; + +__device__ __forceinline__ float sigmoid_fast(float x) { + return 1.0f / (1.0f + __expf(-x)); +} + +__device__ __forceinline__ float silu_fast(float x) { + return x * sigmoid_fast(x); +} + +__device__ __forceinline__ float softplus_fast(float x) { + return x > 20.0f ? x : log1pf(__expf(x)); +} + +__device__ __forceinline__ float load_dt_bias(const void* dt_bias, int head, + int dt_bias_type) { + if (dt_bias_type == kDtBiasBFloat16) { + return __bfloat162float(static_cast(dt_bias)[head]); + } + if (dt_bias_type == kDtBiasFloat16) { + return __half2float(static_cast(dt_bias)[head]); + } + return static_cast(dt_bias)[head]; +} + +__device__ __forceinline__ float warp_reduce_sum(float value) { +#pragma unroll + for (int offset = 16; offset > 0; offset >>= 1) { + value += __shfl_xor_sync(0xffffffffu, value, offset); + } + return value; +} + +struct Sum2 { + float x; + float y; +}; + +__device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) { +#pragma unroll + for (int offset = 16; offset > 0; offset >>= 1) { + x += __shfl_xor_sync(0xffffffffu, x, offset); + y += __shfl_xor_sync(0xffffffffu, y, offset); + } + return {x, y}; +} + +template +__global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel( + const __nv_bfloat16* __restrict__ mixed_qkv, + const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b, + const float* __restrict__ a_log, const void* __restrict__ dt_bias, + const int* __restrict__ state_indices, const int* __restrict__ cu_seqlens, + const int* __restrict__ num_accepted_tokens, StateT* __restrict__ state, + const __nv_bfloat16* __restrict__ output_gate, + const void* __restrict__ norm_weight, __nv_bfloat16* __restrict__ out, + int H, int HV, int state_indices_width, int dt_bias_type, + bool norm_weight_is_bf16, float scale, float norm_eps, + GdnDecodeStrides strides) { + const int request = blockIdx.x; + const int value_head = blockIdx.y; + const int tid = threadIdx.x; + const int lane = tid & 31; + const int warp = tid >> 5; + const int bos = cu_seqlens[request]; + const int eos = cu_seqlens[request + 1]; + const int num_tokens = eos - bos; + if (num_tokens <= 0) { + return; + } + + const int accepted = num_accepted_tokens[request]; + const int source_slot = + accepted > 0 && accepted <= state_indices_width + ? state_indices[request * state_indices_width + accepted - 1] + : 0; + if (source_slot <= 0 || num_tokens > kMaxMtpTokens) { + for (int linear = tid; linear < num_tokens * kDimV; linear += kThreads) { + const int token = bos + linear / kDimV; + const int value = linear % kDimV; + const int64_t out_offset = + (static_cast(token) * HV + value_head) * kDimV + value; + out[out_offset] = __float2bfloat16(0.0f); + } + return; + } + + const int key_head = value_head / 8; + __shared__ StateT shared_state[2][kChunkV][kDimK]; + __shared__ float shared_q[kMaxMtpTokens][kDimK]; + __shared__ float shared_k[kMaxMtpTokens][kDimK]; + __shared__ __nv_bfloat16 shared_v[kMaxMtpTokens][kDimV]; + __shared__ __nv_bfloat16 shared_out[kMaxMtpTokens][kDimV]; + __shared__ float shared_decay[kMaxMtpTokens]; + __shared__ float shared_beta[kMaxMtpTokens]; + + StateT* source_state = + state + static_cast(source_slot) * strides.state_slot + + value_head * kDimV * kDimK; + copy_state_chunk(&shared_state[0][0][0], + source_state, 0, tid, kThreads); + + if (warp < num_tokens) { + const int t = warp; + const int token = bos + t; + const int64_t mixed_base = static_cast(token) * strides.mixed_row; + float q_values[4]; + float k_values[4]; + float q_square = 0.0f; + float k_square = 0.0f; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const int dim = lane + i * 32; + q_values[i] = + __bfloat162float(mixed_qkv[mixed_base + key_head * kDimK + dim]); + k_values[i] = __bfloat162float( + mixed_qkv[mixed_base + H * kDimK + key_head * kDimK + dim]); + shared_v[t][dim] = + mixed_qkv[mixed_base + 2 * H * kDimK + value_head * kDimV + dim]; + q_square += q_values[i] * q_values[i]; + k_square += k_values[i] * k_values[i]; + } + const Sum2 qk_sums = warp_reduce_sum_pair(q_square, k_square); + const float q_scale = __shfl_sync( + 0xffffffffu, lane == 0 ? rsqrtf(qk_sums.x + 1.0e-6f) * scale : 0.0f, 0); + const float k_scale = __shfl_sync( + 0xffffffffu, lane == 0 ? rsqrtf(qk_sums.y + 1.0e-6f) : 0.0f, 0); +#pragma unroll + for (int i = 0; i < 4; ++i) { + const int dim = lane + i * 32; + shared_q[t][dim] = q_values[i] * q_scale; + shared_k[t][dim] = k_values[i] * k_scale; + } + if (lane == 0) { + const float a_value = __bfloat162float( + a[static_cast(token) * strides.a_row + value_head]); + const float b_value = __bfloat162float( + b[static_cast(token) * strides.b_row + value_head]); + const float g = -__expf(a_log[value_head]) * + softplus_fast(a_value + load_dt_bias(dt_bias, value_head, + dt_bias_type)); + shared_decay[t] = __expf(g); + shared_beta[t] = sigmoid_fast(b_value); + } + } + __syncthreads(); + + const int k_base = lane * 4; + int rows[kRowsPerWarp]; +#pragma unroll + for (int row = 0; row < kRowsPerWarp; ++row) { + rows[row] = warp + row * kWarps; + } + +#pragma unroll + for (int chunk = 0; chunk < kNumChunks; ++chunk) { + cp_async_wait_all(); + __syncthreads(); + if (chunk + 1 < kNumChunks) { + copy_state_chunk( + &shared_state[0][0][0], source_state, chunk + 1, tid, kThreads); + } + + float h[kRowsPerWarp][4]; +#pragma unroll + for (int row = 0; row < kRowsPerWarp; ++row) { + const float4 state_value = + load_state4(&shared_state[chunk & 1][rows[row]][k_base]); + h[row][0] = state_value.x; + h[row][1] = state_value.y; + h[row][2] = state_value.z; + h[row][3] = state_value.w; + } + + for (int t = 0; t < num_tokens; ++t) { + const float4 q4 = *reinterpret_cast(&shared_q[t][k_base]); + const float4 k4 = *reinterpret_cast(&shared_k[t][k_base]); + const float q_values[4] = {q4.x, q4.y, q4.z, q4.w}; + const float k_values[4] = {k4.x, k4.y, k4.z, k4.w}; + + float dot_hk[kRowsPerWarp] = {0.0f, 0.0f, 0.0f, 0.0f}; +#pragma unroll + for (int row = 0; row < kRowsPerWarp; ++row) { +#pragma unroll + for (int i = 0; i < 4; ++i) { + h[row][i] *= shared_decay[t]; + dot_hk[row] += h[row][i] * k_values[i]; + } + } + const Sum2 dot_hk_01 = warp_reduce_sum_pair(dot_hk[0], dot_hk[1]); + const Sum2 dot_hk_23 = warp_reduce_sum_pair(dot_hk[2], dot_hk[3]); + const float reduced_hk[kRowsPerWarp] = {dot_hk_01.x, dot_hk_01.y, + dot_hk_23.x, dot_hk_23.y}; + + float dot_hq[kRowsPerWarp] = {0.0f, 0.0f, 0.0f, 0.0f}; +#pragma unroll + for (int row = 0; row < kRowsPerWarp; ++row) { + const int value = chunk * kChunkV + rows[row]; + const float delta = + (__bfloat162float(shared_v[t][value]) - reduced_hk[row]) * + shared_beta[t]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + h[row][i] += k_values[i] * delta; + dot_hq[row] += h[row][i] * q_values[i]; + } + } + const Sum2 dot_hq_01 = warp_reduce_sum_pair(dot_hq[0], dot_hq[1]); + const Sum2 dot_hq_23 = warp_reduce_sum_pair(dot_hq[2], dot_hq[3]); + if (lane == 0) { + shared_out[t][chunk * kChunkV + rows[0]] = + __float2bfloat16(dot_hq_01.x); + shared_out[t][chunk * kChunkV + rows[1]] = + __float2bfloat16(dot_hq_01.y); + shared_out[t][chunk * kChunkV + rows[2]] = + __float2bfloat16(dot_hq_23.x); + shared_out[t][chunk * kChunkV + rows[3]] = + __float2bfloat16(dot_hq_23.y); + } + + const int destination_slot = + state_indices[request * state_indices_width + t]; + if (destination_slot > 0) { + StateT* destination_state = + state + + static_cast(destination_slot) * strides.state_slot + + value_head * kDimV * kDimK; +#pragma unroll + for (int row = 0; row < kRowsPerWarp; ++row) { + const int value = chunk * kChunkV + rows[row]; + store_state4(destination_state + value * kDimK + k_base, + make_float4(h[row][0], h[row][1], h[row][2], h[row][3])); + } + } + } + } + __syncthreads(); + + if (warp < num_tokens) { + const int t = warp; + float output_values[4]; + float sum_square = 0.0f; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const int value = lane + i * 32; + output_values[i] = __bfloat162float(shared_out[t][value]); + sum_square += output_values[i] * output_values[i]; + } + sum_square = warp_reduce_sum(sum_square); + const float rstd = + rsqrtf(sum_square / static_cast(kDimV) + norm_eps); + const int token = bos + t; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const int value = lane + i * 32; + const float gate = silu_fast(__bfloat162float( + output_gate[static_cast(token) * strides.gate_row + + value_head * kDimV + value])); + const float weight = + norm_weight_is_bf16 + ? __bfloat162float( + static_cast(norm_weight)[value]) + : static_cast(norm_weight)[value]; + const int64_t out_offset = + (static_cast(token) * HV + value_head) * kDimV + value; + out[out_offset] = + __float2bfloat16(output_values[i] * rstd * weight * gate); + } + } +} + +template +void launch_gdn_decode_post_conv_mtp( + torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log, + torch::stable::Tensor const& dt_bias, + torch::stable::Tensor const& state_indices, + torch::stable::Tensor const& cu_seqlens, + torch::stable::Tensor const& num_accepted_tokens, + torch::stable::Tensor& state, torch::stable::Tensor const& norm_weight, + torch::stable::Tensor& out, const __nv_bfloat16* a, const __nv_bfloat16* b, + const __nv_bfloat16* output_gate, int num_key_heads, int num_value_heads, + double scale, double norm_eps, GdnDecodeStrides strides) { + using torch::headeronly::ScalarType; + + const auto dt_bias_scalar_type = dt_bias.scalar_type(); + const int dt_bias_type = + dt_bias_scalar_type == ScalarType::Float + ? kDtBiasFloat32 + : (dt_bias_scalar_type == ScalarType::BFloat16 ? kDtBiasBFloat16 + : kDtBiasFloat16); + torch::stable::accelerator::DeviceGuard const device_guard( + mixed_qkv.get_device_index()); + const cudaStream_t stream = + get_current_cuda_stream(mixed_qkv.get_device_index()); + const int num_requests = static_cast(state_indices.size(0)); + const dim3 grid(num_requests, num_value_heads); + gdn_decode_post_conv_mtp_kernel<<>>( + static_cast(mixed_qkv.data_ptr()), a, b, + static_cast(a_log.data_ptr()), dt_bias.data_ptr(), + static_cast(state_indices.data_ptr()), + static_cast(cu_seqlens.data_ptr()), + static_cast(num_accepted_tokens.data_ptr()), + static_cast(state.data_ptr()), output_gate, + norm_weight.data_ptr(), static_cast<__nv_bfloat16*>(out.data_ptr()), + num_key_heads, num_value_heads, static_cast(state_indices.size(1)), + dt_bias_type, norm_weight.scalar_type() == ScalarType::BFloat16, + static_cast(scale), static_cast(norm_eps), strides); + const cudaError_t error = cudaGetLastError(); + STD_TORCH_CHECK(error == cudaSuccess, + "GDN decode MTP post-conv kernel launch failed: ", + cudaGetErrorString(error)); +} + +} // namespace + +void fused_gdn_decode_post_conv_mtp( + torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a, + torch::stable::Tensor const& b, torch::stable::Tensor const& a_log, + torch::stable::Tensor const& dt_bias, + torch::stable::Tensor const& state_indices, + torch::stable::Tensor const& cu_seqlens, + torch::stable::Tensor const& num_accepted_tokens, + torch::stable::Tensor& state, torch::stable::Tensor const& output_gate, + torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out, + double scale, double norm_eps) { + using torch::headeronly::ScalarType; + + STD_TORCH_CHECK( + mixed_qkv.is_cuda() && mixed_qkv.scalar_type() == ScalarType::BFloat16, + "mixed_qkv must be a CUDA bfloat16 tensor"); + STD_TORCH_CHECK(a.is_cuda() && a.scalar_type() == ScalarType::BFloat16, + "a must be a CUDA bfloat16 tensor"); + STD_TORCH_CHECK(b.is_cuda() && b.scalar_type() == ScalarType::BFloat16, + "b must be a CUDA bfloat16 tensor"); + STD_TORCH_CHECK(a_log.is_cuda() && a_log.scalar_type() == ScalarType::Float, + "A_log must be a CUDA float32 tensor"); + const auto dt_bias_scalar_type = dt_bias.scalar_type(); + STD_TORCH_CHECK( + dt_bias.is_cuda() && (dt_bias_scalar_type == ScalarType::Float || + dt_bias_scalar_type == ScalarType::BFloat16 || + dt_bias_scalar_type == ScalarType::Half), + "dt_bias must be a CUDA float32, bfloat16, or float16 tensor"); + STD_TORCH_CHECK( + state_indices.is_cuda() && state_indices.scalar_type() == ScalarType::Int, + "state_indices must be a CUDA int32 tensor"); + STD_TORCH_CHECK( + cu_seqlens.is_cuda() && cu_seqlens.scalar_type() == ScalarType::Int, + "cu_seqlens must be a CUDA int32 tensor"); + STD_TORCH_CHECK(num_accepted_tokens.is_cuda() && + num_accepted_tokens.scalar_type() == ScalarType::Int, + "num_accepted_tokens must be a CUDA int32 tensor"); + const auto state_scalar_type = state.scalar_type(); + STD_TORCH_CHECK( + state.is_cuda() && (state_scalar_type == ScalarType::Float || + state_scalar_type == ScalarType::BFloat16), + "state must be a CUDA float32 or bfloat16 tensor"); + STD_TORCH_CHECK(output_gate.is_cuda() && + output_gate.scalar_type() == ScalarType::BFloat16, + "output_gate must be a CUDA bfloat16 tensor"); + STD_TORCH_CHECK(norm_weight.is_cuda() && + (norm_weight.scalar_type() == ScalarType::Float || + norm_weight.scalar_type() == ScalarType::BFloat16), + "norm_weight must be a CUDA float32 or bfloat16 tensor"); + STD_TORCH_CHECK(out.is_cuda() && out.scalar_type() == ScalarType::BFloat16, + "out must be a CUDA bfloat16 tensor"); + + STD_TORCH_CHECK(mixed_qkv.dim() == 2, + "mixed_qkv must have shape [L, 2 * H * 128 + HV * 128]"); + const int num_tokens = static_cast(mixed_qkv.size(0)); + STD_TORCH_CHECK(num_tokens > 0, + "GDN decode MTP fusion requires at least one token"); + STD_TORCH_CHECK( + state.dim() == 4 && state.size(2) == kDimV && state.size(3) == kDimK, + "state must have shape [slots, HV, 128, 128]"); + const int num_value_heads = static_cast(state.size(1)); + const int64_t key_width = + mixed_qkv.size(1) - static_cast(num_value_heads) * kDimV; + STD_TORCH_CHECK(key_width > 0 && key_width % (2 * kDimK) == 0, + "mixed_qkv width is inconsistent with state"); + const int num_key_heads = static_cast(key_width / (2 * kDimK)); + STD_TORCH_CHECK(num_value_heads == 8 * num_key_heads, + "GDN decode MTP fusion requires HV/H=8"); + + STD_TORCH_CHECK(state_indices.dim() == 2 && state_indices.size(0) > 0 && + state_indices.size(1) > 0 && + state_indices.size(1) <= kMaxMtpTokens, + "state_indices must have shape [N, S] with 1 <= S <= 8"); + const int num_requests = static_cast(state_indices.size(0)); + STD_TORCH_CHECK( + cu_seqlens.dim() == 1 && cu_seqlens.numel() == num_requests + 1, + "cu_seqlens must have N + 1 elements"); + STD_TORCH_CHECK(num_accepted_tokens.dim() == 1 && + num_accepted_tokens.numel() == num_requests, + "num_accepted_tokens must have N elements"); + STD_TORCH_CHECK( + a.dim() == 2 && a.size(0) == num_tokens && a.size(1) == num_value_heads, + "a must have shape [L, HV]"); + STD_TORCH_CHECK( + b.dim() == 2 && b.size(0) == num_tokens && b.size(1) == num_value_heads, + "b must have shape [L, HV]"); + STD_TORCH_CHECK(a_log.is_contiguous() && a_log.numel() == num_value_heads, + "A_log must be contiguous with HV elements"); + STD_TORCH_CHECK(dt_bias.is_contiguous() && dt_bias.numel() == num_value_heads, + "dt_bias must be contiguous with HV elements"); + STD_TORCH_CHECK(state_indices.is_contiguous(), + "state_indices must be contiguous"); + STD_TORCH_CHECK(cu_seqlens.is_contiguous(), "cu_seqlens must be contiguous"); + STD_TORCH_CHECK(num_accepted_tokens.is_contiguous(), + "num_accepted_tokens must be contiguous"); + STD_TORCH_CHECK(output_gate.dim() == 3 && output_gate.size(0) == num_tokens && + output_gate.size(1) == num_value_heads && + output_gate.size(2) == kDimV, + "output_gate must have shape [L, HV, 128]"); + STD_TORCH_CHECK(norm_weight.is_contiguous() && norm_weight.numel() == kDimV, + "norm_weight must be contiguous with 128 elements"); + STD_TORCH_CHECK(out.dim() == 3 && out.size(0) == num_tokens && + out.size(1) == num_value_heads && out.size(2) == kDimV, + "out must have shape [L, HV, 128]"); + STD_TORCH_CHECK(mixed_qkv.stride(1) == 1, + "mixed_qkv channels must be contiguous"); + STD_TORCH_CHECK(a.stride(1) == 1 && b.stride(1) == 1, + "a and b heads must be contiguous"); + STD_TORCH_CHECK(state.stride(0) >= num_value_heads * kDimV * kDimK && + state.stride(1) == kDimV * kDimK && + state.stride(2) == kDimK && state.stride(3) == 1, + "state must have contiguous [HV, 128, 128] slot contents"); + const int state_elements_per_copy = + state_scalar_type == ScalarType::Float ? 4 : 8; + STD_TORCH_CHECK(reinterpret_cast(state.data_ptr()) % 16 == 0 && + state.stride(0) % state_elements_per_copy == 0, + "state slots must preserve 16-byte alignment"); + STD_TORCH_CHECK(output_gate.stride(2) == 1 && output_gate.stride(1) == kDimV, + "output_gate head rows must be contiguous"); + STD_TORCH_CHECK(out.is_contiguous(), "out must be contiguous"); + STD_TORCH_CHECK(norm_eps >= 0.0, "norm_eps must be non-negative"); + + const GdnDecodeStrides strides{mixed_qkv.stride(0), a.stride(0), b.stride(0), + output_gate.stride(0), state.stride(0)}; + const auto* a_ptr = static_cast(a.data_ptr()); + const auto* b_ptr = static_cast(b.data_ptr()); + const auto* output_gate_ptr = + static_cast(output_gate.data_ptr()); + if (state_scalar_type == ScalarType::Float) { + launch_gdn_decode_post_conv_mtp( + mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens, + num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr, + output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps, + strides); + } else { + launch_gdn_decode_post_conv_mtp<__nv_bfloat16>( + mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens, + num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr, + output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps, + strides); + } +} diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 85c875402c..5c595bfaf4 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -400,6 +400,18 @@ void fused_kda_decode( torch::stable::Tensor& out, std::optional lower_bound, std::optional output_gate, std::optional norm_weight, double norm_eps); + +void fused_gdn_decode_post_conv_mtp( + torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a, + torch::stable::Tensor const& b, torch::stable::Tensor const& a_log, + torch::stable::Tensor const& dt_bias, + torch::stable::Tensor const& state_indices, + torch::stable::Tensor const& cu_seqlens, + torch::stable::Tensor const& num_accepted_tokens, + torch::stable::Tensor& state, torch::stable::Tensor const& output_gate, + torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out, + double scale, double norm_eps); + #endif #ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 801aed03d9..28090ce5dc 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -534,6 +534,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "Tensor? norm_weight=None, float norm_eps=1e-5) -> ()"); #endif +#ifdef VLLM_ENABLE_FUSED_GDN_DECODE + ops.def( + "fused_gdn_decode_post_conv_mtp(" + "Tensor mixed_qkv, Tensor a, Tensor b, Tensor A_log, Tensor dt_bias, " + "Tensor state_indices, Tensor cu_seqlens, Tensor num_accepted_tokens, " + "Tensor! state, Tensor output_gate, Tensor norm_weight, Tensor! out, " + "float scale, float norm_eps=1e-5) -> ()"); +#endif + #ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES ops.def( "kimi_k3_attn_res(" @@ -797,6 +806,11 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) { ops.impl("fused_kda_decode", TORCH_BOX(&fused_kda_decode)); #endif +#ifdef VLLM_ENABLE_FUSED_GDN_DECODE + ops.impl("fused_gdn_decode_post_conv_mtp", + TORCH_BOX(&fused_gdn_decode_post_conv_mtp)); +#endif + #ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res)); #endif diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 84a7733f18..39b9912041 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -252,6 +252,11 @@ def test_splitting_ops_dynamic(): # populated when the engine decides to use piecewise compilation. assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE assert config.compilation_config.splitting_ops_contain_attention() + splitting_ops = config.compilation_config.splitting_ops + assert splitting_ops is not None + assert { + "vllm::qwen_gdn_attention_core_fused_norm_packed", + } <= set(splitting_ops) # When use_inductor_graph_partition=True config = VllmConfig( diff --git a/tests/kernels/mamba/test_gdn_fused_mtp.py b/tests/kernels/mamba/test_gdn_fused_mtp.py new file mode 100644 index 0000000000..97ed4c48e7 --- /dev/null +++ b/tests/kernels/mamba/test_gdn_fused_mtp.py @@ -0,0 +1,345 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Model-path tests for fused Qwen3.5 GDN MTP decode.""" + +from __future__ import annotations + +import types +from unittest.mock import patch + +import pytest +import torch + +from vllm.platforms import current_platform + +if not (current_platform.is_cuda() and current_platform.has_device_capability(80)): + pytest.skip( + reason="Fused GDN MTP decode requires CUDA compute capability 8.0+.", + allow_module_level=True, + ) + +from tests.v1.attention.utils import ( # noqa: E402 + BatchSpec, + create_common_attn_metadata, + create_vllm_config, +) +from vllm.config import SpeculativeConfig, set_current_vllm_config # noqa: E402 +from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn # noqa: E402 +from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( # noqa: E402 + ChunkGatedDeltaRule, + QwenGatedDeltaNetAttention, +) +from vllm.model_executor.layers.mamba.mamba_utils import ( # noqa: E402 + MambaStateShapeCalculator, +) +from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( # noqa: E402 + rmsnorm_fn, +) +from vllm.utils.torch_utils import _encode_layer_name # noqa: E402 +from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402 + GDNAttentionMetadataBuilder, +) +from vllm.v1.kv_cache_interface import MambaSpec # noqa: E402 + +NUM_SPEC = 3 +SPEC_TOKENS = NUM_SPEC + 1 +H = 1 +HV = 8 +K = 128 +V = 128 +CONV_KERNEL = 4 +CONV_DIM = 2 * H * K + HV * V +BLOCK_SIZE = 16 +PREFIX = "model.layers.0.linear_attn" +EPS = 1e-6 + + +class _TestGatedNorm: + def __init__(self, weight: torch.Tensor) -> None: + self.weight = weight + self.bias = None + self.eps = EPS + self.group_size = None + self.norm_before_gate = True + self.activation = "silu" + + def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor: + return rmsnorm_fn( + x, + self.weight, + None, + z=z, + eps=self.eps, + norm_before_gate=True, + activation=self.activation, + ) + + +def _make_vllm_config(): + config = create_vllm_config( + model_name="Qwen/Qwen3.5-0.8B", + block_size=BLOCK_SIZE, + hf_config_override={"linear_key_head_dim": K}, + ) + config.additional_config = {"gdn_prefill_backend": "cutedsl"} + config.cache_config.mamba_cache_mode = "none" + config.speculative_config = SpeculativeConfig( + method="ngram", num_speculative_tokens=NUM_SPEC + ) + return config + + +def _build_layer( + vllm_config, + conv_state: torch.Tensor, + ssm_state: torch.Tensor, + a_log: torch.Tensor, + dt_bias: torch.Tensor, + conv_weight: torch.Tensor, + norm_weight: torch.Tensor, +): + layer = types.SimpleNamespace( + prefix=PREFIX, + enable_packed_recurrent_decode=False, + disable_tp_for_ba_proj=False, + tp_size=1, + num_k_heads=H, + num_v_heads=HV, + head_k_dim=K, + head_v_dim=V, + key_dim=K, + value_dim=HV * V, + activation="silu", + A_log=a_log, + dt_bias=dt_bias, + conv1d=types.SimpleNamespace(weight=conv_weight, bias=None), + kv_cache=(conv_state, ssm_state), + norm=_TestGatedNorm(norm_weight), + layer_norm_epsilon=EPS, + gdn_decode_kernel="cuda", + ) + with set_current_vllm_config(vllm_config): + layer.chunk_gated_delta_rule = ChunkGatedDeltaRule() + for name in ( + "rearrange_mixed_qkv", + "_forward_core", + "_forward_core_decode_spec_post_conv_fused_norm", + "_forward_core_decode_spec_fused_norm", + "_can_use_fused_gdn_mtp_decode", + "_rms_norm_gated_cuda", + "_forward_core_fused_norm", + "_forward_core_fused_norm_packed", + "split_ba", + ): + setattr( + layer, + name, + types.MethodType(getattr(QwenGatedDeltaNetAttention, name), layer), + ) + return layer + + +@torch.inference_mode() +def test_fused_forward_uses_packed_entrypoint() -> None: + """Fused mode keeps projected QKVZ and BA packed through the model op.""" + device = torch.device("cuda") + num_tokens = 3 + hidden_states = torch.empty(num_tokens, 1, dtype=torch.bfloat16, device=device) + mixed_qkvz = torch.randn( + num_tokens, CONV_DIM + HV * V, dtype=torch.bfloat16, device=device + ) + ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device) + layer = types.SimpleNamespace( + prefix=PREFIX, + enable_fused_gdn_decode=True, + norm=types.SimpleNamespace( + weight=torch.empty(V, dtype=torch.bfloat16, device=device) + ), + num_v_heads=HV, + tp_size=1, + head_v_dim=V, + in_proj_qkvz=lambda _: (mixed_qkvz, None), + in_proj_ba=lambda _: (ba, None), + out_proj=lambda x: (x, None), + ) + layer.forward_cuda = types.MethodType( + QwenGatedDeltaNetAttention.forward_cuda, layer + ) + + def packed_op( + actual_qkvz: torch.Tensor, + actual_ba: torch.Tensor, + output: torch.Tensor, + *, + layer_name: str, + ) -> None: + assert actual_qkvz is mixed_qkvz + assert actual_ba is ba + assert layer_name == _encode_layer_name(PREFIX) + output.fill_(1) + + with ( + patch.object( + torch.ops.vllm, + "qwen_gdn_attention_core_fused_norm_packed", + side_effect=packed_op, + ) as packed_mock, + patch.object(torch.ops.vllm, "qwen_gdn_attention_core") as triton_mock, + ): + output = layer.forward_cuda(hidden_states) + + packed_mock.assert_called_once() + triton_mock.assert_not_called() + torch.testing.assert_close(output, torch.ones_like(output)) + + +@pytest.mark.parametrize( + "seq_lens,query_lens,draft_tokens,expected_fused_calls", + [ + pytest.param([128], [SPEC_TOKENS], [NUM_SPEC], 1, id="pure-mtp"), + pytest.param( + [128, 96], + [SPEC_TOKENS, 64], + [NUM_SPEC, -1], + 0, + id="mixed-mtp-falls-back", + ), + pytest.param([96], [64], [-1], 0, id="pure-prefill"), + pytest.param([128], [1], [-1], 0, id="pure-decode"), + ], +) +@torch.inference_mode() +def test_fused_model_path_matches_reference( + seq_lens: list[int], + query_lens: list[int], + draft_tokens: list[int], + expected_fused_calls: int, +) -> None: + """Fused MTP and its mixed/prefill/decode fallbacks match the reference.""" + torch.manual_seed(1) + device = torch.device("cuda") + vllm_config = _make_vllm_config() + builder = GDNAttentionMetadataBuilder( + kv_cache_spec=MambaSpec( + block_size=BLOCK_SIZE, + shapes=((16, 64),), + dtypes=(torch.float16,), + num_speculative_blocks=NUM_SPEC, + ), + layer_names=[PREFIX], + vllm_config=vllm_config, + device=device, + ) + batch = BatchSpec(seq_lens=seq_lens, query_lens=query_lens) + common = create_common_attn_metadata( + batch, BLOCK_SIZE, device, arange_block_indices=True + ) + common.block_table_tensor.add_(1) + with set_current_vllm_config(vllm_config): + metadata = builder.build( + common_prefix_len=0, + common_attn_metadata=common, + num_accepted_tokens=torch.ones( + batch.batch_size, dtype=torch.int32, device=device + ), + num_decode_draft_tokens_cpu=torch.tensor(draft_tokens, dtype=torch.int32), + ) + + state_indices = [ + indices + for indices in ( + metadata.spec_state_indices_tensor, + metadata.non_spec_state_indices_tensor, + ) + if indices is not None and indices.numel() > 0 + ] + pool_size = max(int(indices.max().item()) for indices in state_indices) + 1 + conv_state_shape, temporal_state_shape = ( + MambaStateShapeCalculator.gated_delta_net_state_shape( + 1, H, HV, K, V, CONV_KERNEL, NUM_SPEC + ) + ) + conv_state_seed = 0.05 * torch.randn( + pool_size, *conv_state_shape, dtype=torch.bfloat16, device=device + ) + ssm_state_seed = 0.01 * torch.randn( + pool_size, *temporal_state_shape, dtype=torch.float32, device=device + ) + a_log = 0.1 * torch.randn(HV, dtype=torch.float32, device=device) + dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device) + conv_weight = 0.1 * torch.randn( + CONV_DIM, 1, CONV_KERNEL, dtype=torch.bfloat16, device=device + ) + norm_weight = torch.randn(V, dtype=torch.float32, device=device) + num_tokens = batch.compute_num_tokens() + mixed_qkv = 0.1 * torch.randn( + num_tokens, CONV_DIM, dtype=torch.bfloat16, device=device + ) + b = 0.1 * torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device) + a = 0.1 * torch.randn_like(b) + output_gate = 0.1 * torch.randn( + num_tokens, HV, V, dtype=torch.bfloat16, device=device + ) + mixed_qkvz = torch.cat((mixed_qkv, output_gate.flatten(1)), dim=-1) + ba = torch.cat((b, a), dim=-1) + context = types.SimpleNamespace(attn_metadata={PREFIX: metadata}) + + reference_layer = _build_layer( + vllm_config, + conv_state_seed.clone(), + ssm_state_seed.clone(), + a_log, + dt_bias, + conv_weight, + norm_weight, + ) + reference_out = torch.zeros_like(output_gate) + with patch.object( + qwen_gdn_linear_attn, "get_forward_context", return_value=context + ): + reference_layer._forward_core( + mixed_qkv=mixed_qkv.clone(), + b=b, + a=a, + core_attn_out=reference_out, + ) + reference_out = reference_layer.norm(reference_out, output_gate) + + fused_layer = _build_layer( + vllm_config, + conv_state_seed.clone(), + ssm_state_seed.clone(), + a_log, + dt_bias, + conv_weight, + norm_weight, + ) + context.no_compile_layers = {PREFIX: fused_layer} + fused_out = torch.zeros_like(output_gate) + fused_op = qwen_gdn_linear_attn.ops.fused_gdn_decode_post_conv_mtp + with ( + patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=context), + patch.object( + qwen_gdn_linear_attn.ops, + "fused_gdn_decode_post_conv_mtp", + wraps=fused_op, + ) as fused_mock, + ): + torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed( + mixed_qkvz.clone(), + ba, + fused_out, + layer_name=_encode_layer_name(PREFIX), + ) + + assert fused_mock.call_count == expected_fused_calls + torch.testing.assert_close( + fused_layer.kv_cache[0], reference_layer.kv_cache[0], atol=0, rtol=0 + ) + torch.testing.assert_close(fused_out, reference_out, atol=3e-2, rtol=3e-2) + torch.testing.assert_close( + fused_layer.kv_cache[1], + reference_layer.kv_cache[1], + atol=3e-2, + rtol=3e-2, + ) diff --git a/tests/kernels/test_fused_gdn_post_conv.py b/tests/kernels/test_fused_gdn_post_conv.py index fad77891a4..b6dacf2e73 100644 --- a/tests/kernels/test_fused_gdn_post_conv.py +++ b/tests/kernels/test_fused_gdn_post_conv.py @@ -1,18 +1,25 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for fused_gdn_prefill_post_conv kernel. +"""Tests for GDN post-convolution kernels. -Verifies that the fused kernel matches the reference: - split → rearrange → contiguous → l2norm → gating +The prefill tests cover preparation and the MTP tests cover recurrent state +updates plus output normalization and gating. """ import pytest import torch import torch.nn.functional as F +from vllm import _custom_ops as ops +from vllm.third_party.flash_linear_attention.ops import ( + fused_sigmoid_gating_delta_rule_update, +) from vllm.third_party.flash_linear_attention.ops.fused_gdn_prefill_post_conv import ( fused_post_conv_prep, ) +from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( + rmsnorm_fn, +) def reference_post_conv( @@ -207,3 +214,132 @@ def test_fused_post_conv_l0(): ) assert q.shape == (0, H, K) assert g.shape == (0, HV) + + +@pytest.mark.parametrize( + "tp_size,query_lengths,state_dtype,norm_dtype", + [ + pytest.param(16, (4, 4), torch.bfloat16, torch.bfloat16, id="tp16-bf16"), + pytest.param(4, (4, 4), torch.float32, torch.float32, id="tp4-fp32"), + pytest.param(16, (4, 2, 0), torch.bfloat16, torch.float32, id="tp16-ragged"), + pytest.param(4, (4, 2, 0), torch.float32, torch.bfloat16, id="tp4-ragged"), + pytest.param(16, (8,), torch.float32, torch.bfloat16, id="tp16-max"), + pytest.param(4, (8,), torch.bfloat16, torch.float32, id="tp4-max"), + ], +) +@torch.inference_mode() +def test_fused_gdn_decode_post_conv_mtp_ratio8( + tp_size: int, + query_lengths: tuple[int, ...], + state_dtype: torch.dtype, + norm_dtype: torch.dtype, +) -> None: + if torch.cuda.get_device_capability() < (8, 0): + pytest.skip("fused GDN decode MTP requires compute capability 8.0+") + if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"): + pytest.skip("fused GDN decode MTP op is not built") + + torch.manual_seed(0) + device = "cuda" + H = 16 // tp_size + HV = 128 // tp_size + K = V = 128 + num_reqs = len(query_lengths) + state_width = max(query_lengths) + num_tokens = sum(query_lengths) + num_slots = num_reqs * state_width + 1 + scale = K**-0.5 + eps = 1e-6 + + mixed_qkv = torch.randn( + num_tokens, + 2 * H * K + HV * V, + dtype=torch.bfloat16, + device=device, + ) + query, key, value = torch.split( + mixed_qkv, + [H * K, H * K, HV * V], + dim=-1, + ) + query = query.view(1, num_tokens, H, K) + key = key.view(1, num_tokens, H, K) + value = value.view(1, num_tokens, HV, V) + ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device) + b, a = ba.chunk(2, dim=-1) + assert not a.is_contiguous() + assert not b.is_contiguous() + A_log = 0.5 * torch.randn(HV, dtype=torch.float32, device=device) + dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device) + output_gate = torch.randn(num_tokens, HV, V, dtype=torch.bfloat16, device=device) + norm_weight = torch.randn(V, dtype=norm_dtype, device=device) + state_ref = ( + 0.01 * torch.randn(num_slots, HV, V, K, dtype=torch.float32, device=device) + ).to(state_dtype) + state_actual = state_ref.clone() + state_indices = torch.arange(1, num_slots, dtype=torch.int32, device=device).view( + num_reqs, state_width + ) + cu_seqlens = torch.tensor( + [0, *torch.tensor(query_lengths).cumsum(0).tolist()], + dtype=torch.int32, + device=device, + ) + num_accepted_tokens = torch.ones(num_reqs, dtype=torch.int32, device=device) + if query_lengths[-1] == 0: + state_indices[-1].zero_() + + for step, accepted_tokens in enumerate((1, min(2, state_width), state_width)): + num_accepted_tokens.fill_(accepted_tokens) + if query_lengths[-1] == 0: + num_accepted_tokens[-1] = 1 + raw_ref, _ = fused_sigmoid_gating_delta_rule_update( + A_log=A_log, + a=a, + b=b, + dt_bias=dt_bias, + q=query, + k=key, + v=value, + initial_state=state_ref, + inplace_final_state=True, + cu_seqlens=cu_seqlens, + ssm_state_indices=state_indices, + num_accepted_tokens=num_accepted_tokens, + scale=scale, + use_qk_l2norm_in_kernel=True, + ) + expected = rmsnorm_fn( + raw_ref.squeeze(0), + norm_weight, + None, + z=output_gate, + eps=eps, + norm_before_gate=True, + activation="silu", + ) + actual = ops.fused_gdn_decode_post_conv_mtp( + mixed_qkv=mixed_qkv, + a=a, + b=b, + A_log=A_log, + dt_bias=dt_bias, + state_indices=state_indices, + cu_seqlens=cu_seqlens, + num_accepted_tokens=num_accepted_tokens, + state=state_actual, + output_gate=output_gate, + norm_weight=norm_weight, + out=torch.empty_like(output_gate), + scale=scale, + norm_eps=eps, + ) + + output_error = (actual.float() - expected.float()).norm() + output_relative_l2 = output_error / expected.float().norm().clamp_min(1e-20) + assert output_relative_l2 < 5e-4, ( + f"MTP output relative L2 mismatch at step {step}: " + f"{output_relative_l2.item():.6g}" + ) + + torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2) diff --git a/tests/test_envs.py b/tests/test_envs.py index 5e0363e33a..3d214fdbff 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -251,6 +251,21 @@ class TestEnvWithChoices: env_func() +def test_gdn_decode_kernel_env(monkeypatch: pytest.MonkeyPatch): + env_func = environment_variables["VLLM_GDN_DECODE_KERNEL"] + monkeypatch.delenv("VLLM_GDN_DECODE_KERNEL", raising=False) + assert env_func() == "cuda" + + for value in ("cuda", "triton"): + monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value) + assert env_func() == value + + for value in ("fused", "invalid"): + monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value) + with pytest.raises(ValueError, match="VLLM_GDN_DECODE_KERNEL"): + env_func() + + class TestEnvListWithChoices: """Test cases for env_list_with_choices function.""" diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index c4ff05c75d..5d63ffa0de 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2777,6 +2777,43 @@ def fused_kda_decode( return out +def fused_gdn_decode_post_conv_mtp( + mixed_qkv: torch.Tensor, + a: torch.Tensor, + b: torch.Tensor, + A_log: torch.Tensor, + dt_bias: torch.Tensor, + state_indices: torch.Tensor, + cu_seqlens: torch.Tensor, + num_accepted_tokens: torch.Tensor, + state: torch.Tensor, + output_gate: torch.Tensor, + norm_weight: torch.Tensor, + out: torch.Tensor | None = None, + scale: float = 128**-0.5, + norm_eps: float = 1e-5, +) -> torch.Tensor: + if out is None: + out = torch.empty_like(output_gate) + torch.ops._C.fused_gdn_decode_post_conv_mtp( + mixed_qkv, + a, + b, + A_log, + dt_bias, + state_indices, + cu_seqlens, + num_accepted_tokens, + state, + output_gate, + norm_weight, + out, + scale, + norm_eps, + ) + return out + + def concat_and_cache_mla( kv_c: torch.Tensor, k_pe: torch.Tensor, diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 0776a60a46..3cd227d72c 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -769,6 +769,7 @@ class CompilationConfig: "vllm::short_conv", "vllm::linear_attention", "vllm::qwen_gdn_attention_core", + "vllm::qwen_gdn_attention_core_fused_norm_packed", "vllm::gdn_attention_core_xpu", "vllm::olmo_hybrid_gdn_full_forward", "vllm::sparse_attn_indexer", diff --git a/vllm/envs.py b/vllm/envs.py index 5715508d12..e68e1a335e 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -127,6 +127,7 @@ if TYPE_CHECKING: VLLM_DISABLED_KERNELS: list[str] = [] VLLM_USE_HW_AGNOSTIC: bool = False VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True + VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda" VLLM_DISABLE_PYNCCL: bool = False VLLM_USE_OINK_OPS: bool = False VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD: bool = True @@ -1209,6 +1210,15 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool( int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1")) ), + # Select the GDN MTP decode implementation. "cuda" uses the fused decode + # kernel where supported and falls back to "triton" otherwise; setting it + # explicitly to "cuda" raises when unsupported. + "VLLM_GDN_DECODE_KERNEL": env_with_choices( + "VLLM_GDN_DECODE_KERNEL", + "cuda", + ["cuda", "triton"], + case_sensitive=False, + ), # Disable pynccl (using torch.distributed instead) "VLLM_DISABLE_PYNCCL": lambda: ( os.getenv("VLLM_DISABLE_PYNCCL", "False").lower() in ("true", "1") diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index f3ebcd6325..27dc4d90d7 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -2,12 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Inference-only Qwen3-Next/Qwen3.5 model.""" +import os from typing import Literal import torch from einops import rearrange from torch import nn +from vllm import _custom_ops as ops from vllm import envs from vllm._aiter_ops import rocm_aiter_ops from vllm.config import ( @@ -84,6 +86,9 @@ if GDN_AITER_TRITON_AVAILABLE: logger = init_logger(__name__) +MAX_FUSED_GDN_MTP_TOKENS = 8 +FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16) + def _resolve_gdn_prefill_backend( vllm_config: VllmConfig, @@ -484,12 +489,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): self.enable_packed_recurrent_decode = ( envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE ) + self.gdn_decode_kernel = envs.VLLM_GDN_DECODE_KERNEL.strip().lower() + if self.gdn_decode_kernel == "cuda": + reason = self._fused_gdn_decode_unsupported_reason(vllm_config) + if reason is not None: + if "VLLM_GDN_DECODE_KERNEL" in os.environ: + raise ValueError( + f"VLLM_GDN_DECODE_KERNEL=cuda is not supported: {reason}" + ) + logger.info_once( + "Falling back to the Triton GDN decode path: %s", reason + ) + self.gdn_decode_kernel = "triton" + self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda" + logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel) compilation_config = get_current_vllm_config().compilation_config if prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self + def _fused_gdn_decode_unsupported_reason( + self, vllm_config: VllmConfig + ) -> str | None: + conv_state_dtype, recurrent_state_dtype = self.get_state_dtype() + if ( + self.gqa_interleaved_layout + or self.head_k_dim != 128 + or self.head_v_dim != 128 + or self.norm.activation != "silu" + or vllm_config.model_config.dtype != torch.bfloat16 + or conv_state_dtype != torch.bfloat16 + or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES + or not current_platform.has_device_capability(80) + ): + return ( + "the fused CUDA kernel requires a BF16 GDN model with " + "K=V=128, SiLU gating, non-interleaved GQA layout, BF16 " + "convolution cache, BF16 or FP32 recurrent state, and a " + "GPU with compute capability 8.0+" + ) + if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"): + return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built" + return None + def create_qkvz_proj( self, hidden_size: int, @@ -846,6 +889,26 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): mixed_qkvz, _ = self.in_proj_qkvz(hidden_states) ba, _ = self.in_proj_ba(hidden_states) + use_fused_gdn_decode = ( + self.enable_fused_gdn_decode + and hidden_states.dtype == torch.bfloat16 + and self.norm.weight.dtype in (torch.bfloat16, torch.float32) + ) + if use_fused_gdn_decode: + core_attn_out = torch.zeros( + (num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim), + dtype=hidden_states.dtype, + device=hidden_states.device, + ) + torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed( + mixed_qkvz, + ba, + core_attn_out, + layer_name=_encode_layer_name(self.prefix), + ) + output, _ = self.out_proj(core_attn_out.flatten(-2)) + return output + if self.gqa_interleaved_layout: # Qwen3-Next: unpack the interleaved GQA layout query, key, value, z, b, a = self.fix_query_key_value_ordering( @@ -862,8 +925,6 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): mixed_qkv, z = mixed_qkvz.split([qkv_size, z_size], dim=-1) z = z.reshape(z.size(0), -1, self.head_v_dim) b, a = self.split_ba(ba) - b = b.contiguous() - a = a.contiguous() # ============================================================ # Part 2: Core Attention (Custom Op) @@ -878,8 +939,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): torch.ops.vllm.qwen_gdn_attention_core( mixed_qkv, - b, - a, + b.contiguous(), + a.contiguous(), core_attn_out, layer_name=_encode_layer_name(self.prefix), ) @@ -1622,6 +1683,211 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): ) return + def _forward_core_decode_spec_fused_norm( + self, + mixed_qkv: torch.Tensor, + b: torch.Tensor, + a: torch.Tensor, + output_gate: torch.Tensor, + core_attn_out: torch.Tensor, + attn_metadata: GDNAttentionMetadata, + ) -> None: + state_indices = attn_metadata.spec_state_indices_tensor + cu_seqlens = attn_metadata.spec_query_start_loc + num_accepted_tokens = attn_metadata.num_accepted_tokens + assert state_indices is not None + assert cu_seqlens is not None + assert num_accepted_tokens is not None + + num_requests = attn_metadata.num_spec_decodes + num_actual_tokens = attn_metadata.num_actual_tokens + conv_state = ( + self.kv_cache[0] + if is_conv_state_dim_first() + else self.kv_cache[0].transpose(-1, -2) + ) + conv_weights = self.conv1d.weight.view( + self.conv1d.weight.size(0), self.conv1d.weight.size(2) + ) + mixed_qkv = causal_conv1d_update( + mixed_qkv[:num_actual_tokens], + conv_state, + conv_weights, + self.conv1d.bias, + self.activation, + conv_state_indices=state_indices[:num_requests, 0], + num_accepted_tokens=num_accepted_tokens[:num_requests], + query_start_loc=cu_seqlens[: num_requests + 1], + max_query_len=state_indices.size(1), + validate_data=False, + ) + self._forward_core_decode_spec_post_conv_fused_norm( + mixed_qkv=mixed_qkv, + b=b[:num_actual_tokens], + a=a[:num_actual_tokens], + output_gate=output_gate[:num_actual_tokens], + core_attn_out=core_attn_out[:num_actual_tokens], + attn_metadata=attn_metadata, + ) + + def _forward_core_decode_spec_post_conv_fused_norm( + self, + mixed_qkv: torch.Tensor, + b: torch.Tensor, + a: torch.Tensor, + output_gate: torch.Tensor, + core_attn_out: torch.Tensor, + attn_metadata: GDNAttentionMetadata, + ) -> None: + state_indices = attn_metadata.spec_state_indices_tensor + cu_seqlens = attn_metadata.spec_query_start_loc + num_accepted_tokens = attn_metadata.num_accepted_tokens + assert state_indices is not None + assert cu_seqlens is not None + assert num_accepted_tokens is not None + + num_requests = attn_metadata.num_spec_decodes + ops.fused_gdn_decode_post_conv_mtp( + mixed_qkv=mixed_qkv, + a=a, + b=b, + A_log=self.A_log, + dt_bias=self.dt_bias, + state_indices=state_indices[:num_requests], + cu_seqlens=cu_seqlens[: num_requests + 1], + num_accepted_tokens=num_accepted_tokens[:num_requests], + state=self.kv_cache[1], + output_gate=output_gate, + norm_weight=self.norm.weight, + out=core_attn_out, + scale=self.head_k_dim**-0.5, + norm_eps=self.layer_norm_epsilon, + ) + + def _forward_core_fused_norm_packed( + self, + mixed_qkvz: torch.Tensor, + ba: torch.Tensor, + core_attn_out: torch.Tensor, + ) -> None: + forward_context = get_forward_context() + attn_metadata_raw = forward_context.attn_metadata + qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size + if attn_metadata_raw is None: + self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0) + return + + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index] + assert isinstance(attn_metadata, GDNAttentionMetadata) + mixed_qkv, output_gate_flat = mixed_qkvz.split( + [qkv_size, self.value_dim // self.tp_size], dim=-1 + ) + output_gate = output_gate_flat.reshape( + output_gate_flat.size(0), -1, self.head_v_dim + ) + b, a = self.split_ba(ba) + self._forward_core_fused_norm( + mixed_qkv=mixed_qkv, + b=b, + a=a, + output_gate=output_gate, + core_attn_out=core_attn_out, + ) + + def _can_use_fused_gdn_mtp_decode( + self, attn_metadata: GDNAttentionMetadata + ) -> bool: + state_indices = attn_metadata.spec_state_indices_tensor + return ( + attn_metadata.spec_sequence_masks is not None + and attn_metadata.num_decodes == 0 + and attn_metadata.num_spec_decodes > 0 + and self.kv_cache[1].dtype in FUSED_GDN_STATE_DTYPES + and self.gdn_decode_kernel == "cuda" + and self.num_v_heads == 8 * self.num_k_heads + and state_indices is not None + and state_indices.size(1) <= MAX_FUSED_GDN_MTP_TOKENS + and hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp") + ) + + def _rms_norm_gated_cuda( + self, + x: torch.Tensor, + output_gate: torch.Tensor, + out: torch.Tensor, + ) -> None: + from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( + layer_norm_fwd, + ) + + x_shape = x.shape + assert output_gate.shape == x_shape + assert out.shape == x_shape + x_2d = x.reshape(-1, x_shape[-1]) + output_gate_2d = output_gate.reshape(-1, x_shape[-1]) + out_2d = out.reshape(-1, x_shape[-1]) + assert x_2d.stride(-1) == 1 + assert output_gate_2d.stride(-1) == 1 + assert out_2d.stride(-1) == 1 + layer_norm_fwd( + x_2d, + self.norm.weight.contiguous(), + self.norm.bias, + self.norm.eps, + z=output_gate_2d, + out=out_2d, + group_size=( + x_shape[-1] if self.norm.group_size is None else self.norm.group_size + ), + norm_before_gate=self.norm.norm_before_gate, + is_rms_norm=True, + activation=self.norm.activation, + ) + + def _forward_core_fused_norm( + self, + mixed_qkv: torch.Tensor, + b: torch.Tensor, + a: torch.Tensor, + output_gate: torch.Tensor, + core_attn_out: torch.Tensor, + ) -> None: + forward_context = get_forward_context() + attn_metadata_raw = forward_context.attn_metadata + if attn_metadata_raw is None: + self._warmup_prefill_kernels(mixed_qkv, 0) + return + + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index] + assert isinstance(attn_metadata, GDNAttentionMetadata) + if ( + self._can_use_fused_gdn_mtp_decode(attn_metadata) + and attn_metadata.num_prefills == 0 + ): + self._forward_core_decode_spec_fused_norm( + mixed_qkv=mixed_qkv, + b=b, + a=a, + output_gate=output_gate, + core_attn_out=core_attn_out, + attn_metadata=attn_metadata, + ) + return + self._forward_core( + mixed_qkv=mixed_qkv, + b=b.contiguous(), + a=a.contiguous(), + core_attn_out=core_attn_out, + ) + num_actual_tokens = attn_metadata.num_actual_tokens + self._rms_norm_gated_cuda( + core_attn_out[:num_actual_tokens], + output_gate[:num_actual_tokens], + core_attn_out[:num_actual_tokens], + ) + def qwen_gdn_attention_core( qkv_or_qkvz: torch.Tensor, @@ -1683,6 +1949,39 @@ direct_register_custom_op( ) +def qwen_gdn_attention_core_fused_norm_packed( + mixed_qkvz: torch.Tensor, + ba: torch.Tensor, + core_attn_out: torch.Tensor, + layer_name: LayerNameType, +) -> None: + layer_name = _resolve_layer_name(layer_name) + forward_context: ForwardContext = get_forward_context() + self = forward_context.no_compile_layers[layer_name] + self._forward_core_fused_norm_packed( + mixed_qkvz=mixed_qkvz, + ba=ba, + core_attn_out=core_attn_out, + ) + + +def gdn_attention_core_fused_norm_packed_fake( + mixed_qkvz: torch.Tensor, + ba: torch.Tensor, + core_attn_out: torch.Tensor, + layer_name: LayerNameType, +) -> None: + return + + +direct_register_custom_op( + op_name="qwen_gdn_attention_core_fused_norm_packed", + op_func=qwen_gdn_attention_core_fused_norm_packed, + mutates_args=["core_attn_out"], + fake_impl=gdn_attention_core_fused_norm_packed_fake, +) + + @triton.jit def fused_gdn_gating_kernel( g,