[Kernel][Perf] Add fused CUDA post-conv MTP decode kernel for Qwen3.5 GDN (#51674)
Signed-off-by: Jie Fang <jief@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Fable 5
parent
d18bb7b390
commit
1be3628367
@@ -1118,6 +1118,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
|
||||
cuda_archs_loose_intersection(FUSED_KDA_DECODE_ARCHS
|
||||
"9.0a;10.0f;12.0f" "${CUDA_ARCHS}")
|
||||
cuda_archs_loose_intersection(FUSED_GDN_DECODE_ARCHS
|
||||
"8.0;8.6;8.9;9.0a;10.0f;12.0f" "${CUDA_ARCHS}")
|
||||
endif()
|
||||
if(FUSED_KDA_DECODE_ARCHS)
|
||||
set(FUSED_KDA_DECODE_SRC
|
||||
@@ -1131,6 +1133,18 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
message(STATUS
|
||||
"Building fused KDA decode for archs: ${FUSED_KDA_DECODE_ARCHS}")
|
||||
endif()
|
||||
if(FUSED_GDN_DECODE_ARCHS)
|
||||
set(FUSED_GDN_DECODE_SRC
|
||||
"csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu")
|
||||
set_gencode_flags_for_srcs(
|
||||
SRCS "${FUSED_GDN_DECODE_SRC}"
|
||||
CUDA_ARCHS "${FUSED_GDN_DECODE_ARCHS}")
|
||||
set_property(SOURCE ${FUSED_GDN_DECODE_SRC} APPEND PROPERTY
|
||||
COMPILE_OPTIONS "$<$<COMPILE_LANGUAGE:CUDA>:--use_fast_math>")
|
||||
list(APPEND VLLM_STABLE_EXT_SRC "${FUSED_GDN_DECODE_SRC}")
|
||||
message(STATUS
|
||||
"Building fused GDN decode for archs: ${FUSED_GDN_DECODE_ARCHS}")
|
||||
endif()
|
||||
|
||||
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
|
||||
cuda_archs_loose_intersection(KIMI_K3_ATTN_RES_ARCHS
|
||||
@@ -1213,6 +1227,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
target_compile_definitions(_C_stable_libtorch PRIVATE
|
||||
VLLM_ENABLE_FUSED_KDA_DECODE=1)
|
||||
endif()
|
||||
if(FUSED_GDN_DECODE_ARCHS)
|
||||
target_compile_definitions(_C_stable_libtorch PRIVATE
|
||||
VLLM_ENABLE_FUSED_GDN_DECODE=1)
|
||||
endif()
|
||||
if(KIMI_K3_ATTN_RES_ARCHS)
|
||||
target_compile_definitions(_C_stable_libtorch PRIVATE
|
||||
VLLM_ENABLE_KIMI_K3_ATTN_RES=1)
|
||||
|
||||
@@ -0,0 +1,557 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
*/
|
||||
|
||||
#include <cstdint>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include "../torch_utils.h"
|
||||
#include "../../cuda_compat.h"
|
||||
|
||||
namespace {
|
||||
|
||||
template <typename StateT>
|
||||
__device__ __forceinline__ void cp_async_16b(StateT* smem_ptr,
|
||||
const StateT* gmem_ptr) {
|
||||
const uint32_t smem_addr =
|
||||
static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
|
||||
asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n"
|
||||
:
|
||||
: "r"(smem_addr), "l"(gmem_ptr));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cp_async_commit() {
|
||||
asm volatile("cp.async.commit_group;\n" ::);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cp_async_wait_all() {
|
||||
asm volatile("cp.async.wait_all;\n" ::: "memory");
|
||||
}
|
||||
|
||||
template <typename StateT, int ChunkV, int DimK, int Stages>
|
||||
__device__ __forceinline__ void copy_state_chunk(StateT* shared_state,
|
||||
const StateT* state, int chunk,
|
||||
int thread, int threads) {
|
||||
constexpr int kElementsPerCopy = 16 / sizeof(StateT);
|
||||
constexpr int kCopiesPerChunk = ChunkV * DimK / kElementsPerCopy;
|
||||
const int stage = chunk % Stages;
|
||||
for (int copy = thread; copy < kCopiesPerChunk; copy += threads) {
|
||||
const int element = copy * kElementsPerCopy;
|
||||
cp_async_16b(shared_state + stage * ChunkV * DimK + element,
|
||||
state + chunk * ChunkV * DimK + element);
|
||||
}
|
||||
cp_async_commit();
|
||||
}
|
||||
|
||||
template <typename StateT>
|
||||
__device__ __forceinline__ float4 load_state4(const StateT* state);
|
||||
|
||||
template <>
|
||||
__device__ __forceinline__ float4 load_state4<float>(const float* state) {
|
||||
return *reinterpret_cast<const float4*>(state);
|
||||
}
|
||||
|
||||
template <>
|
||||
__device__ __forceinline__ float4
|
||||
load_state4<__nv_bfloat16>(const __nv_bfloat16* state) {
|
||||
const __nv_bfloat162 lo = *reinterpret_cast<const __nv_bfloat162*>(state);
|
||||
const __nv_bfloat162 hi = *reinterpret_cast<const __nv_bfloat162*>(state + 2);
|
||||
return make_float4(__bfloat162float(lo.x), __bfloat162float(lo.y),
|
||||
__bfloat162float(hi.x), __bfloat162float(hi.y));
|
||||
}
|
||||
|
||||
template <typename StateT>
|
||||
__device__ __forceinline__ void store_state4(StateT* state, float4 value);
|
||||
|
||||
template <>
|
||||
__device__ __forceinline__ void store_state4<float>(float* state,
|
||||
float4 value) {
|
||||
*reinterpret_cast<float4*>(state) = value;
|
||||
}
|
||||
|
||||
template <>
|
||||
__device__ __forceinline__ void store_state4<__nv_bfloat16>(
|
||||
__nv_bfloat16* state, float4 value) {
|
||||
*reinterpret_cast<__nv_bfloat162*>(state) =
|
||||
__floats2bfloat162_rn(value.x, value.y);
|
||||
*reinterpret_cast<__nv_bfloat162*>(state + 2) =
|
||||
__floats2bfloat162_rn(value.z, value.w);
|
||||
}
|
||||
|
||||
constexpr int kDimK = 128;
|
||||
constexpr int kDimV = 128;
|
||||
constexpr int kThreads = 256;
|
||||
constexpr int kWarps = kThreads / 32;
|
||||
constexpr int kChunkV = 32;
|
||||
constexpr int kNumChunks = kDimV / kChunkV;
|
||||
constexpr int kRowsPerWarp = kChunkV / kWarps;
|
||||
constexpr int kMaxMtpTokens = 8;
|
||||
constexpr int kDtBiasFloat32 = 0;
|
||||
constexpr int kDtBiasBFloat16 = 1;
|
||||
constexpr int kDtBiasFloat16 = 2;
|
||||
|
||||
struct GdnDecodeStrides {
|
||||
int64_t mixed_row;
|
||||
int64_t a_row;
|
||||
int64_t b_row;
|
||||
int64_t gate_row;
|
||||
int64_t state_slot;
|
||||
};
|
||||
|
||||
__device__ __forceinline__ float sigmoid_fast(float x) {
|
||||
return 1.0f / (1.0f + __expf(-x));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float silu_fast(float x) {
|
||||
return x * sigmoid_fast(x);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float softplus_fast(float x) {
|
||||
return x > 20.0f ? x : log1pf(__expf(x));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float load_dt_bias(const void* dt_bias, int head,
|
||||
int dt_bias_type) {
|
||||
if (dt_bias_type == kDtBiasBFloat16) {
|
||||
return __bfloat162float(static_cast<const __nv_bfloat16*>(dt_bias)[head]);
|
||||
}
|
||||
if (dt_bias_type == kDtBiasFloat16) {
|
||||
return __half2float(static_cast<const __half*>(dt_bias)[head]);
|
||||
}
|
||||
return static_cast<const float*>(dt_bias)[head];
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float warp_reduce_sum(float value) {
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset >>= 1) {
|
||||
value += __shfl_xor_sync(0xffffffffu, value, offset);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
struct Sum2 {
|
||||
float x;
|
||||
float y;
|
||||
};
|
||||
|
||||
__device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) {
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset >>= 1) {
|
||||
x += __shfl_xor_sync(0xffffffffu, x, offset);
|
||||
y += __shfl_xor_sync(0xffffffffu, y, offset);
|
||||
}
|
||||
return {x, y};
|
||||
}
|
||||
|
||||
template <typename StateT>
|
||||
__global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
const __nv_bfloat16* __restrict__ mixed_qkv,
|
||||
const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b,
|
||||
const float* __restrict__ a_log, const void* __restrict__ dt_bias,
|
||||
const int* __restrict__ state_indices, const int* __restrict__ cu_seqlens,
|
||||
const int* __restrict__ num_accepted_tokens, StateT* __restrict__ state,
|
||||
const __nv_bfloat16* __restrict__ output_gate,
|
||||
const void* __restrict__ norm_weight, __nv_bfloat16* __restrict__ out,
|
||||
int H, int HV, int state_indices_width, int dt_bias_type,
|
||||
bool norm_weight_is_bf16, float scale, float norm_eps,
|
||||
GdnDecodeStrides strides) {
|
||||
const int request = blockIdx.x;
|
||||
const int value_head = blockIdx.y;
|
||||
const int tid = threadIdx.x;
|
||||
const int lane = tid & 31;
|
||||
const int warp = tid >> 5;
|
||||
const int bos = cu_seqlens[request];
|
||||
const int eos = cu_seqlens[request + 1];
|
||||
const int num_tokens = eos - bos;
|
||||
if (num_tokens <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int accepted = num_accepted_tokens[request];
|
||||
const int source_slot =
|
||||
accepted > 0 && accepted <= state_indices_width
|
||||
? state_indices[request * state_indices_width + accepted - 1]
|
||||
: 0;
|
||||
if (source_slot <= 0 || num_tokens > kMaxMtpTokens) {
|
||||
for (int linear = tid; linear < num_tokens * kDimV; linear += kThreads) {
|
||||
const int token = bos + linear / kDimV;
|
||||
const int value = linear % kDimV;
|
||||
const int64_t out_offset =
|
||||
(static_cast<int64_t>(token) * HV + value_head) * kDimV + value;
|
||||
out[out_offset] = __float2bfloat16(0.0f);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const int key_head = value_head / 8;
|
||||
__shared__ StateT shared_state[2][kChunkV][kDimK];
|
||||
__shared__ float shared_q[kMaxMtpTokens][kDimK];
|
||||
__shared__ float shared_k[kMaxMtpTokens][kDimK];
|
||||
__shared__ __nv_bfloat16 shared_v[kMaxMtpTokens][kDimV];
|
||||
__shared__ __nv_bfloat16 shared_out[kMaxMtpTokens][kDimV];
|
||||
__shared__ float shared_decay[kMaxMtpTokens];
|
||||
__shared__ float shared_beta[kMaxMtpTokens];
|
||||
|
||||
StateT* source_state =
|
||||
state + static_cast<int64_t>(source_slot) * strides.state_slot +
|
||||
value_head * kDimV * kDimK;
|
||||
copy_state_chunk<StateT, kChunkV, kDimK, 2>(&shared_state[0][0][0],
|
||||
source_state, 0, tid, kThreads);
|
||||
|
||||
if (warp < num_tokens) {
|
||||
const int t = warp;
|
||||
const int token = bos + t;
|
||||
const int64_t mixed_base = static_cast<int64_t>(token) * strides.mixed_row;
|
||||
float q_values[4];
|
||||
float k_values[4];
|
||||
float q_square = 0.0f;
|
||||
float k_square = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
const int dim = lane + i * 32;
|
||||
q_values[i] =
|
||||
__bfloat162float(mixed_qkv[mixed_base + key_head * kDimK + dim]);
|
||||
k_values[i] = __bfloat162float(
|
||||
mixed_qkv[mixed_base + H * kDimK + key_head * kDimK + dim]);
|
||||
shared_v[t][dim] =
|
||||
mixed_qkv[mixed_base + 2 * H * kDimK + value_head * kDimV + dim];
|
||||
q_square += q_values[i] * q_values[i];
|
||||
k_square += k_values[i] * k_values[i];
|
||||
}
|
||||
const Sum2 qk_sums = warp_reduce_sum_pair(q_square, k_square);
|
||||
const float q_scale = __shfl_sync(
|
||||
0xffffffffu, lane == 0 ? rsqrtf(qk_sums.x + 1.0e-6f) * scale : 0.0f, 0);
|
||||
const float k_scale = __shfl_sync(
|
||||
0xffffffffu, lane == 0 ? rsqrtf(qk_sums.y + 1.0e-6f) : 0.0f, 0);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
const int dim = lane + i * 32;
|
||||
shared_q[t][dim] = q_values[i] * q_scale;
|
||||
shared_k[t][dim] = k_values[i] * k_scale;
|
||||
}
|
||||
if (lane == 0) {
|
||||
const float a_value = __bfloat162float(
|
||||
a[static_cast<int64_t>(token) * strides.a_row + value_head]);
|
||||
const float b_value = __bfloat162float(
|
||||
b[static_cast<int64_t>(token) * strides.b_row + value_head]);
|
||||
const float g = -__expf(a_log[value_head]) *
|
||||
softplus_fast(a_value + load_dt_bias(dt_bias, value_head,
|
||||
dt_bias_type));
|
||||
shared_decay[t] = __expf(g);
|
||||
shared_beta[t] = sigmoid_fast(b_value);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int k_base = lane * 4;
|
||||
int rows[kRowsPerWarp];
|
||||
#pragma unroll
|
||||
for (int row = 0; row < kRowsPerWarp; ++row) {
|
||||
rows[row] = warp + row * kWarps;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int chunk = 0; chunk < kNumChunks; ++chunk) {
|
||||
cp_async_wait_all();
|
||||
__syncthreads();
|
||||
if (chunk + 1 < kNumChunks) {
|
||||
copy_state_chunk<StateT, kChunkV, kDimK, 2>(
|
||||
&shared_state[0][0][0], source_state, chunk + 1, tid, kThreads);
|
||||
}
|
||||
|
||||
float h[kRowsPerWarp][4];
|
||||
#pragma unroll
|
||||
for (int row = 0; row < kRowsPerWarp; ++row) {
|
||||
const float4 state_value =
|
||||
load_state4(&shared_state[chunk & 1][rows[row]][k_base]);
|
||||
h[row][0] = state_value.x;
|
||||
h[row][1] = state_value.y;
|
||||
h[row][2] = state_value.z;
|
||||
h[row][3] = state_value.w;
|
||||
}
|
||||
|
||||
for (int t = 0; t < num_tokens; ++t) {
|
||||
const float4 q4 = *reinterpret_cast<const float4*>(&shared_q[t][k_base]);
|
||||
const float4 k4 = *reinterpret_cast<const float4*>(&shared_k[t][k_base]);
|
||||
const float q_values[4] = {q4.x, q4.y, q4.z, q4.w};
|
||||
const float k_values[4] = {k4.x, k4.y, k4.z, k4.w};
|
||||
|
||||
float dot_hk[kRowsPerWarp] = {0.0f, 0.0f, 0.0f, 0.0f};
|
||||
#pragma unroll
|
||||
for (int row = 0; row < kRowsPerWarp; ++row) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
h[row][i] *= shared_decay[t];
|
||||
dot_hk[row] += h[row][i] * k_values[i];
|
||||
}
|
||||
}
|
||||
const Sum2 dot_hk_01 = warp_reduce_sum_pair(dot_hk[0], dot_hk[1]);
|
||||
const Sum2 dot_hk_23 = warp_reduce_sum_pair(dot_hk[2], dot_hk[3]);
|
||||
const float reduced_hk[kRowsPerWarp] = {dot_hk_01.x, dot_hk_01.y,
|
||||
dot_hk_23.x, dot_hk_23.y};
|
||||
|
||||
float dot_hq[kRowsPerWarp] = {0.0f, 0.0f, 0.0f, 0.0f};
|
||||
#pragma unroll
|
||||
for (int row = 0; row < kRowsPerWarp; ++row) {
|
||||
const int value = chunk * kChunkV + rows[row];
|
||||
const float delta =
|
||||
(__bfloat162float(shared_v[t][value]) - reduced_hk[row]) *
|
||||
shared_beta[t];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
h[row][i] += k_values[i] * delta;
|
||||
dot_hq[row] += h[row][i] * q_values[i];
|
||||
}
|
||||
}
|
||||
const Sum2 dot_hq_01 = warp_reduce_sum_pair(dot_hq[0], dot_hq[1]);
|
||||
const Sum2 dot_hq_23 = warp_reduce_sum_pair(dot_hq[2], dot_hq[3]);
|
||||
if (lane == 0) {
|
||||
shared_out[t][chunk * kChunkV + rows[0]] =
|
||||
__float2bfloat16(dot_hq_01.x);
|
||||
shared_out[t][chunk * kChunkV + rows[1]] =
|
||||
__float2bfloat16(dot_hq_01.y);
|
||||
shared_out[t][chunk * kChunkV + rows[2]] =
|
||||
__float2bfloat16(dot_hq_23.x);
|
||||
shared_out[t][chunk * kChunkV + rows[3]] =
|
||||
__float2bfloat16(dot_hq_23.y);
|
||||
}
|
||||
|
||||
const int destination_slot =
|
||||
state_indices[request * state_indices_width + t];
|
||||
if (destination_slot > 0) {
|
||||
StateT* destination_state =
|
||||
state +
|
||||
static_cast<int64_t>(destination_slot) * strides.state_slot +
|
||||
value_head * kDimV * kDimK;
|
||||
#pragma unroll
|
||||
for (int row = 0; row < kRowsPerWarp; ++row) {
|
||||
const int value = chunk * kChunkV + rows[row];
|
||||
store_state4(destination_state + value * kDimK + k_base,
|
||||
make_float4(h[row][0], h[row][1], h[row][2], h[row][3]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
if (warp < num_tokens) {
|
||||
const int t = warp;
|
||||
float output_values[4];
|
||||
float sum_square = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
const int value = lane + i * 32;
|
||||
output_values[i] = __bfloat162float(shared_out[t][value]);
|
||||
sum_square += output_values[i] * output_values[i];
|
||||
}
|
||||
sum_square = warp_reduce_sum(sum_square);
|
||||
const float rstd =
|
||||
rsqrtf(sum_square / static_cast<float>(kDimV) + norm_eps);
|
||||
const int token = bos + t;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
const int value = lane + i * 32;
|
||||
const float gate = silu_fast(__bfloat162float(
|
||||
output_gate[static_cast<int64_t>(token) * strides.gate_row +
|
||||
value_head * kDimV + value]));
|
||||
const float weight =
|
||||
norm_weight_is_bf16
|
||||
? __bfloat162float(
|
||||
static_cast<const __nv_bfloat16*>(norm_weight)[value])
|
||||
: static_cast<const float*>(norm_weight)[value];
|
||||
const int64_t out_offset =
|
||||
(static_cast<int64_t>(token) * HV + value_head) * kDimV + value;
|
||||
out[out_offset] =
|
||||
__float2bfloat16(output_values[i] * rstd * weight * gate);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename StateT>
|
||||
void launch_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log,
|
||||
torch::stable::Tensor const& dt_bias,
|
||||
torch::stable::Tensor const& state_indices,
|
||||
torch::stable::Tensor const& cu_seqlens,
|
||||
torch::stable::Tensor const& num_accepted_tokens,
|
||||
torch::stable::Tensor& state, torch::stable::Tensor const& norm_weight,
|
||||
torch::stable::Tensor& out, const __nv_bfloat16* a, const __nv_bfloat16* b,
|
||||
const __nv_bfloat16* output_gate, int num_key_heads, int num_value_heads,
|
||||
double scale, double norm_eps, GdnDecodeStrides strides) {
|
||||
using torch::headeronly::ScalarType;
|
||||
|
||||
const auto dt_bias_scalar_type = dt_bias.scalar_type();
|
||||
const int dt_bias_type =
|
||||
dt_bias_scalar_type == ScalarType::Float
|
||||
? kDtBiasFloat32
|
||||
: (dt_bias_scalar_type == ScalarType::BFloat16 ? kDtBiasBFloat16
|
||||
: kDtBiasFloat16);
|
||||
torch::stable::accelerator::DeviceGuard const device_guard(
|
||||
mixed_qkv.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(mixed_qkv.get_device_index());
|
||||
const int num_requests = static_cast<int>(state_indices.size(0));
|
||||
const dim3 grid(num_requests, num_value_heads);
|
||||
gdn_decode_post_conv_mtp_kernel<StateT><<<grid, kThreads, 0, stream>>>(
|
||||
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
|
||||
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
|
||||
static_cast<const int*>(state_indices.data_ptr()),
|
||||
static_cast<const int*>(cu_seqlens.data_ptr()),
|
||||
static_cast<const int*>(num_accepted_tokens.data_ptr()),
|
||||
static_cast<StateT*>(state.data_ptr()), output_gate,
|
||||
norm_weight.data_ptr(), static_cast<__nv_bfloat16*>(out.data_ptr()),
|
||||
num_key_heads, num_value_heads, static_cast<int>(state_indices.size(1)),
|
||||
dt_bias_type, norm_weight.scalar_type() == ScalarType::BFloat16,
|
||||
static_cast<float>(scale), static_cast<float>(norm_eps), strides);
|
||||
const cudaError_t error = cudaGetLastError();
|
||||
STD_TORCH_CHECK(error == cudaSuccess,
|
||||
"GDN decode MTP post-conv kernel launch failed: ",
|
||||
cudaGetErrorString(error));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void fused_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a,
|
||||
torch::stable::Tensor const& b, torch::stable::Tensor const& a_log,
|
||||
torch::stable::Tensor const& dt_bias,
|
||||
torch::stable::Tensor const& state_indices,
|
||||
torch::stable::Tensor const& cu_seqlens,
|
||||
torch::stable::Tensor const& num_accepted_tokens,
|
||||
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
|
||||
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
|
||||
double scale, double norm_eps) {
|
||||
using torch::headeronly::ScalarType;
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
mixed_qkv.is_cuda() && mixed_qkv.scalar_type() == ScalarType::BFloat16,
|
||||
"mixed_qkv must be a CUDA bfloat16 tensor");
|
||||
STD_TORCH_CHECK(a.is_cuda() && a.scalar_type() == ScalarType::BFloat16,
|
||||
"a must be a CUDA bfloat16 tensor");
|
||||
STD_TORCH_CHECK(b.is_cuda() && b.scalar_type() == ScalarType::BFloat16,
|
||||
"b must be a CUDA bfloat16 tensor");
|
||||
STD_TORCH_CHECK(a_log.is_cuda() && a_log.scalar_type() == ScalarType::Float,
|
||||
"A_log must be a CUDA float32 tensor");
|
||||
const auto dt_bias_scalar_type = dt_bias.scalar_type();
|
||||
STD_TORCH_CHECK(
|
||||
dt_bias.is_cuda() && (dt_bias_scalar_type == ScalarType::Float ||
|
||||
dt_bias_scalar_type == ScalarType::BFloat16 ||
|
||||
dt_bias_scalar_type == ScalarType::Half),
|
||||
"dt_bias must be a CUDA float32, bfloat16, or float16 tensor");
|
||||
STD_TORCH_CHECK(
|
||||
state_indices.is_cuda() && state_indices.scalar_type() == ScalarType::Int,
|
||||
"state_indices must be a CUDA int32 tensor");
|
||||
STD_TORCH_CHECK(
|
||||
cu_seqlens.is_cuda() && cu_seqlens.scalar_type() == ScalarType::Int,
|
||||
"cu_seqlens must be a CUDA int32 tensor");
|
||||
STD_TORCH_CHECK(num_accepted_tokens.is_cuda() &&
|
||||
num_accepted_tokens.scalar_type() == ScalarType::Int,
|
||||
"num_accepted_tokens must be a CUDA int32 tensor");
|
||||
const auto state_scalar_type = state.scalar_type();
|
||||
STD_TORCH_CHECK(
|
||||
state.is_cuda() && (state_scalar_type == ScalarType::Float ||
|
||||
state_scalar_type == ScalarType::BFloat16),
|
||||
"state must be a CUDA float32 or bfloat16 tensor");
|
||||
STD_TORCH_CHECK(output_gate.is_cuda() &&
|
||||
output_gate.scalar_type() == ScalarType::BFloat16,
|
||||
"output_gate must be a CUDA bfloat16 tensor");
|
||||
STD_TORCH_CHECK(norm_weight.is_cuda() &&
|
||||
(norm_weight.scalar_type() == ScalarType::Float ||
|
||||
norm_weight.scalar_type() == ScalarType::BFloat16),
|
||||
"norm_weight must be a CUDA float32 or bfloat16 tensor");
|
||||
STD_TORCH_CHECK(out.is_cuda() && out.scalar_type() == ScalarType::BFloat16,
|
||||
"out must be a CUDA bfloat16 tensor");
|
||||
|
||||
STD_TORCH_CHECK(mixed_qkv.dim() == 2,
|
||||
"mixed_qkv must have shape [L, 2 * H * 128 + HV * 128]");
|
||||
const int num_tokens = static_cast<int>(mixed_qkv.size(0));
|
||||
STD_TORCH_CHECK(num_tokens > 0,
|
||||
"GDN decode MTP fusion requires at least one token");
|
||||
STD_TORCH_CHECK(
|
||||
state.dim() == 4 && state.size(2) == kDimV && state.size(3) == kDimK,
|
||||
"state must have shape [slots, HV, 128, 128]");
|
||||
const int num_value_heads = static_cast<int>(state.size(1));
|
||||
const int64_t key_width =
|
||||
mixed_qkv.size(1) - static_cast<int64_t>(num_value_heads) * kDimV;
|
||||
STD_TORCH_CHECK(key_width > 0 && key_width % (2 * kDimK) == 0,
|
||||
"mixed_qkv width is inconsistent with state");
|
||||
const int num_key_heads = static_cast<int>(key_width / (2 * kDimK));
|
||||
STD_TORCH_CHECK(num_value_heads == 8 * num_key_heads,
|
||||
"GDN decode MTP fusion requires HV/H=8");
|
||||
|
||||
STD_TORCH_CHECK(state_indices.dim() == 2 && state_indices.size(0) > 0 &&
|
||||
state_indices.size(1) > 0 &&
|
||||
state_indices.size(1) <= kMaxMtpTokens,
|
||||
"state_indices must have shape [N, S] with 1 <= S <= 8");
|
||||
const int num_requests = static_cast<int>(state_indices.size(0));
|
||||
STD_TORCH_CHECK(
|
||||
cu_seqlens.dim() == 1 && cu_seqlens.numel() == num_requests + 1,
|
||||
"cu_seqlens must have N + 1 elements");
|
||||
STD_TORCH_CHECK(num_accepted_tokens.dim() == 1 &&
|
||||
num_accepted_tokens.numel() == num_requests,
|
||||
"num_accepted_tokens must have N elements");
|
||||
STD_TORCH_CHECK(
|
||||
a.dim() == 2 && a.size(0) == num_tokens && a.size(1) == num_value_heads,
|
||||
"a must have shape [L, HV]");
|
||||
STD_TORCH_CHECK(
|
||||
b.dim() == 2 && b.size(0) == num_tokens && b.size(1) == num_value_heads,
|
||||
"b must have shape [L, HV]");
|
||||
STD_TORCH_CHECK(a_log.is_contiguous() && a_log.numel() == num_value_heads,
|
||||
"A_log must be contiguous with HV elements");
|
||||
STD_TORCH_CHECK(dt_bias.is_contiguous() && dt_bias.numel() == num_value_heads,
|
||||
"dt_bias must be contiguous with HV elements");
|
||||
STD_TORCH_CHECK(state_indices.is_contiguous(),
|
||||
"state_indices must be contiguous");
|
||||
STD_TORCH_CHECK(cu_seqlens.is_contiguous(), "cu_seqlens must be contiguous");
|
||||
STD_TORCH_CHECK(num_accepted_tokens.is_contiguous(),
|
||||
"num_accepted_tokens must be contiguous");
|
||||
STD_TORCH_CHECK(output_gate.dim() == 3 && output_gate.size(0) == num_tokens &&
|
||||
output_gate.size(1) == num_value_heads &&
|
||||
output_gate.size(2) == kDimV,
|
||||
"output_gate must have shape [L, HV, 128]");
|
||||
STD_TORCH_CHECK(norm_weight.is_contiguous() && norm_weight.numel() == kDimV,
|
||||
"norm_weight must be contiguous with 128 elements");
|
||||
STD_TORCH_CHECK(out.dim() == 3 && out.size(0) == num_tokens &&
|
||||
out.size(1) == num_value_heads && out.size(2) == kDimV,
|
||||
"out must have shape [L, HV, 128]");
|
||||
STD_TORCH_CHECK(mixed_qkv.stride(1) == 1,
|
||||
"mixed_qkv channels must be contiguous");
|
||||
STD_TORCH_CHECK(a.stride(1) == 1 && b.stride(1) == 1,
|
||||
"a and b heads must be contiguous");
|
||||
STD_TORCH_CHECK(state.stride(0) >= num_value_heads * kDimV * kDimK &&
|
||||
state.stride(1) == kDimV * kDimK &&
|
||||
state.stride(2) == kDimK && state.stride(3) == 1,
|
||||
"state must have contiguous [HV, 128, 128] slot contents");
|
||||
const int state_elements_per_copy =
|
||||
state_scalar_type == ScalarType::Float ? 4 : 8;
|
||||
STD_TORCH_CHECK(reinterpret_cast<uintptr_t>(state.data_ptr()) % 16 == 0 &&
|
||||
state.stride(0) % state_elements_per_copy == 0,
|
||||
"state slots must preserve 16-byte alignment");
|
||||
STD_TORCH_CHECK(output_gate.stride(2) == 1 && output_gate.stride(1) == kDimV,
|
||||
"output_gate head rows must be contiguous");
|
||||
STD_TORCH_CHECK(out.is_contiguous(), "out must be contiguous");
|
||||
STD_TORCH_CHECK(norm_eps >= 0.0, "norm_eps must be non-negative");
|
||||
|
||||
const GdnDecodeStrides strides{mixed_qkv.stride(0), a.stride(0), b.stride(0),
|
||||
output_gate.stride(0), state.stride(0)};
|
||||
const auto* a_ptr = static_cast<const __nv_bfloat16*>(a.data_ptr());
|
||||
const auto* b_ptr = static_cast<const __nv_bfloat16*>(b.data_ptr());
|
||||
const auto* output_gate_ptr =
|
||||
static_cast<const __nv_bfloat16*>(output_gate.data_ptr());
|
||||
if (state_scalar_type == ScalarType::Float) {
|
||||
launch_gdn_decode_post_conv_mtp<float>(
|
||||
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
|
||||
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
|
||||
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
|
||||
strides);
|
||||
} else {
|
||||
launch_gdn_decode_post_conv_mtp<__nv_bfloat16>(
|
||||
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
|
||||
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
|
||||
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
|
||||
strides);
|
||||
}
|
||||
}
|
||||
@@ -400,6 +400,18 @@ void fused_kda_decode(
|
||||
torch::stable::Tensor& out, std::optional<double> lower_bound,
|
||||
std::optional<torch::stable::Tensor> output_gate,
|
||||
std::optional<torch::stable::Tensor> norm_weight, double norm_eps);
|
||||
|
||||
void fused_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a,
|
||||
torch::stable::Tensor const& b, torch::stable::Tensor const& a_log,
|
||||
torch::stable::Tensor const& dt_bias,
|
||||
torch::stable::Tensor const& state_indices,
|
||||
torch::stable::Tensor const& cu_seqlens,
|
||||
torch::stable::Tensor const& num_accepted_tokens,
|
||||
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
|
||||
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
|
||||
double scale, double norm_eps);
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
|
||||
@@ -534,6 +534,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor? norm_weight=None, float norm_eps=1e-5) -> ()");
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_FUSED_GDN_DECODE
|
||||
ops.def(
|
||||
"fused_gdn_decode_post_conv_mtp("
|
||||
"Tensor mixed_qkv, Tensor a, Tensor b, Tensor A_log, Tensor dt_bias, "
|
||||
"Tensor state_indices, Tensor cu_seqlens, Tensor num_accepted_tokens, "
|
||||
"Tensor! state, Tensor output_gate, Tensor norm_weight, Tensor! out, "
|
||||
"float scale, float norm_eps=1e-5) -> ()");
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.def(
|
||||
"kimi_k3_attn_res("
|
||||
@@ -797,6 +806,11 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("fused_kda_decode", TORCH_BOX(&fused_kda_decode));
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_FUSED_GDN_DECODE
|
||||
ops.impl("fused_gdn_decode_post_conv_mtp",
|
||||
TORCH_BOX(&fused_gdn_decode_post_conv_mtp));
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res));
|
||||
#endif
|
||||
|
||||
@@ -252,6 +252,11 @@ def test_splitting_ops_dynamic():
|
||||
# populated when the engine decides to use piecewise compilation.
|
||||
assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
assert config.compilation_config.splitting_ops_contain_attention()
|
||||
splitting_ops = config.compilation_config.splitting_ops
|
||||
assert splitting_ops is not None
|
||||
assert {
|
||||
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
||||
} <= set(splitting_ops)
|
||||
|
||||
# When use_inductor_graph_partition=True
|
||||
config = VllmConfig(
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-path tests for fused Qwen3.5 GDN MTP decode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not (current_platform.is_cuda() and current_platform.has_device_capability(80)):
|
||||
pytest.skip(
|
||||
reason="Fused GDN MTP decode requires CUDA compute capability 8.0+.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from tests.v1.attention.utils import ( # noqa: E402
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig, set_current_vllm_config # noqa: E402
|
||||
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn # noqa: E402
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( # noqa: E402
|
||||
ChunkGatedDeltaRule,
|
||||
QwenGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import ( # noqa: E402
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( # noqa: E402
|
||||
rmsnorm_fn,
|
||||
)
|
||||
from vllm.utils.torch_utils import _encode_layer_name # noqa: E402
|
||||
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec # noqa: E402
|
||||
|
||||
NUM_SPEC = 3
|
||||
SPEC_TOKENS = NUM_SPEC + 1
|
||||
H = 1
|
||||
HV = 8
|
||||
K = 128
|
||||
V = 128
|
||||
CONV_KERNEL = 4
|
||||
CONV_DIM = 2 * H * K + HV * V
|
||||
BLOCK_SIZE = 16
|
||||
PREFIX = "model.layers.0.linear_attn"
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
class _TestGatedNorm:
|
||||
def __init__(self, weight: torch.Tensor) -> None:
|
||||
self.weight = weight
|
||||
self.bias = None
|
||||
self.eps = EPS
|
||||
self.group_size = None
|
||||
self.norm_before_gate = True
|
||||
self.activation = "silu"
|
||||
|
||||
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
|
||||
return rmsnorm_fn(
|
||||
x,
|
||||
self.weight,
|
||||
None,
|
||||
z=z,
|
||||
eps=self.eps,
|
||||
norm_before_gate=True,
|
||||
activation=self.activation,
|
||||
)
|
||||
|
||||
|
||||
def _make_vllm_config():
|
||||
config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
hf_config_override={"linear_key_head_dim": K},
|
||||
)
|
||||
config.additional_config = {"gdn_prefill_backend": "cutedsl"}
|
||||
config.cache_config.mamba_cache_mode = "none"
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=NUM_SPEC
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _build_layer(
|
||||
vllm_config,
|
||||
conv_state: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
a_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
):
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
enable_packed_recurrent_decode=False,
|
||||
disable_tp_for_ba_proj=False,
|
||||
tp_size=1,
|
||||
num_k_heads=H,
|
||||
num_v_heads=HV,
|
||||
head_k_dim=K,
|
||||
head_v_dim=V,
|
||||
key_dim=K,
|
||||
value_dim=HV * V,
|
||||
activation="silu",
|
||||
A_log=a_log,
|
||||
dt_bias=dt_bias,
|
||||
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
|
||||
kv_cache=(conv_state, ssm_state),
|
||||
norm=_TestGatedNorm(norm_weight),
|
||||
layer_norm_epsilon=EPS,
|
||||
gdn_decode_kernel="cuda",
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer.chunk_gated_delta_rule = ChunkGatedDeltaRule()
|
||||
for name in (
|
||||
"rearrange_mixed_qkv",
|
||||
"_forward_core",
|
||||
"_forward_core_decode_spec_post_conv_fused_norm",
|
||||
"_forward_core_decode_spec_fused_norm",
|
||||
"_can_use_fused_gdn_mtp_decode",
|
||||
"_rms_norm_gated_cuda",
|
||||
"_forward_core_fused_norm",
|
||||
"_forward_core_fused_norm_packed",
|
||||
"split_ba",
|
||||
):
|
||||
setattr(
|
||||
layer,
|
||||
name,
|
||||
types.MethodType(getattr(QwenGatedDeltaNetAttention, name), layer),
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_fused_forward_uses_packed_entrypoint() -> None:
|
||||
"""Fused mode keeps projected QKVZ and BA packed through the model op."""
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 3
|
||||
hidden_states = torch.empty(num_tokens, 1, dtype=torch.bfloat16, device=device)
|
||||
mixed_qkvz = torch.randn(
|
||||
num_tokens, CONV_DIM + HV * V, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
enable_fused_gdn_decode=True,
|
||||
norm=types.SimpleNamespace(
|
||||
weight=torch.empty(V, dtype=torch.bfloat16, device=device)
|
||||
),
|
||||
num_v_heads=HV,
|
||||
tp_size=1,
|
||||
head_v_dim=V,
|
||||
in_proj_qkvz=lambda _: (mixed_qkvz, None),
|
||||
in_proj_ba=lambda _: (ba, None),
|
||||
out_proj=lambda x: (x, None),
|
||||
)
|
||||
layer.forward_cuda = types.MethodType(
|
||||
QwenGatedDeltaNetAttention.forward_cuda, layer
|
||||
)
|
||||
|
||||
def packed_op(
|
||||
actual_qkvz: torch.Tensor,
|
||||
actual_ba: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
assert actual_qkvz is mixed_qkvz
|
||||
assert actual_ba is ba
|
||||
assert layer_name == _encode_layer_name(PREFIX)
|
||||
output.fill_(1)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
torch.ops.vllm,
|
||||
"qwen_gdn_attention_core_fused_norm_packed",
|
||||
side_effect=packed_op,
|
||||
) as packed_mock,
|
||||
patch.object(torch.ops.vllm, "qwen_gdn_attention_core") as triton_mock,
|
||||
):
|
||||
output = layer.forward_cuda(hidden_states)
|
||||
|
||||
packed_mock.assert_called_once()
|
||||
triton_mock.assert_not_called()
|
||||
torch.testing.assert_close(output, torch.ones_like(output))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,query_lens,draft_tokens,expected_fused_calls",
|
||||
[
|
||||
pytest.param([128], [SPEC_TOKENS], [NUM_SPEC], 1, id="pure-mtp"),
|
||||
pytest.param(
|
||||
[128, 96],
|
||||
[SPEC_TOKENS, 64],
|
||||
[NUM_SPEC, -1],
|
||||
0,
|
||||
id="mixed-mtp-falls-back",
|
||||
),
|
||||
pytest.param([96], [64], [-1], 0, id="pure-prefill"),
|
||||
pytest.param([128], [1], [-1], 0, id="pure-decode"),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_model_path_matches_reference(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
draft_tokens: list[int],
|
||||
expected_fused_calls: int,
|
||||
) -> None:
|
||||
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
|
||||
torch.manual_seed(1)
|
||||
device = torch.device("cuda")
|
||||
vllm_config = _make_vllm_config()
|
||||
builder = GDNAttentionMetadataBuilder(
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=NUM_SPEC,
|
||||
),
|
||||
layer_names=[PREFIX],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
batch = BatchSpec(seq_lens=seq_lens, query_lens=query_lens)
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
common.block_table_tensor.add_(1)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common,
|
||||
num_accepted_tokens=torch.ones(
|
||||
batch.batch_size, dtype=torch.int32, device=device
|
||||
),
|
||||
num_decode_draft_tokens_cpu=torch.tensor(draft_tokens, dtype=torch.int32),
|
||||
)
|
||||
|
||||
state_indices = [
|
||||
indices
|
||||
for indices in (
|
||||
metadata.spec_state_indices_tensor,
|
||||
metadata.non_spec_state_indices_tensor,
|
||||
)
|
||||
if indices is not None and indices.numel() > 0
|
||||
]
|
||||
pool_size = max(int(indices.max().item()) for indices in state_indices) + 1
|
||||
conv_state_shape, temporal_state_shape = (
|
||||
MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
1, H, HV, K, V, CONV_KERNEL, NUM_SPEC
|
||||
)
|
||||
)
|
||||
conv_state_seed = 0.05 * torch.randn(
|
||||
pool_size, *conv_state_shape, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
ssm_state_seed = 0.01 * torch.randn(
|
||||
pool_size, *temporal_state_shape, dtype=torch.float32, device=device
|
||||
)
|
||||
a_log = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
conv_weight = 0.1 * torch.randn(
|
||||
CONV_DIM, 1, CONV_KERNEL, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
norm_weight = torch.randn(V, dtype=torch.float32, device=device)
|
||||
num_tokens = batch.compute_num_tokens()
|
||||
mixed_qkv = 0.1 * torch.randn(
|
||||
num_tokens, CONV_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
b = 0.1 * torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device)
|
||||
a = 0.1 * torch.randn_like(b)
|
||||
output_gate = 0.1 * torch.randn(
|
||||
num_tokens, HV, V, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
mixed_qkvz = torch.cat((mixed_qkv, output_gate.flatten(1)), dim=-1)
|
||||
ba = torch.cat((b, a), dim=-1)
|
||||
context = types.SimpleNamespace(attn_metadata={PREFIX: metadata})
|
||||
|
||||
reference_layer = _build_layer(
|
||||
vllm_config,
|
||||
conv_state_seed.clone(),
|
||||
ssm_state_seed.clone(),
|
||||
a_log,
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
)
|
||||
reference_out = torch.zeros_like(output_gate)
|
||||
with patch.object(
|
||||
qwen_gdn_linear_attn, "get_forward_context", return_value=context
|
||||
):
|
||||
reference_layer._forward_core(
|
||||
mixed_qkv=mixed_qkv.clone(),
|
||||
b=b,
|
||||
a=a,
|
||||
core_attn_out=reference_out,
|
||||
)
|
||||
reference_out = reference_layer.norm(reference_out, output_gate)
|
||||
|
||||
fused_layer = _build_layer(
|
||||
vllm_config,
|
||||
conv_state_seed.clone(),
|
||||
ssm_state_seed.clone(),
|
||||
a_log,
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
)
|
||||
context.no_compile_layers = {PREFIX: fused_layer}
|
||||
fused_out = torch.zeros_like(output_gate)
|
||||
fused_op = qwen_gdn_linear_attn.ops.fused_gdn_decode_post_conv_mtp
|
||||
with (
|
||||
patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=context),
|
||||
patch.object(
|
||||
qwen_gdn_linear_attn.ops,
|
||||
"fused_gdn_decode_post_conv_mtp",
|
||||
wraps=fused_op,
|
||||
) as fused_mock,
|
||||
):
|
||||
torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed(
|
||||
mixed_qkvz.clone(),
|
||||
ba,
|
||||
fused_out,
|
||||
layer_name=_encode_layer_name(PREFIX),
|
||||
)
|
||||
|
||||
assert fused_mock.call_count == expected_fused_calls
|
||||
torch.testing.assert_close(
|
||||
fused_layer.kv_cache[0], reference_layer.kv_cache[0], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(fused_out, reference_out, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(
|
||||
fused_layer.kv_cache[1],
|
||||
reference_layer.kv_cache[1],
|
||||
atol=3e-2,
|
||||
rtol=3e-2,
|
||||
)
|
||||
@@ -1,18 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for fused_gdn_prefill_post_conv kernel.
|
||||
"""Tests for GDN post-convolution kernels.
|
||||
|
||||
Verifies that the fused kernel matches the reference:
|
||||
split → rearrange → contiguous → l2norm → gating
|
||||
The prefill tests cover preparation and the MTP tests cover recurrent state
|
||||
updates plus output normalization and gating.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.third_party.flash_linear_attention.ops import (
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.fused_gdn_prefill_post_conv import (
|
||||
fused_post_conv_prep,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
rmsnorm_fn,
|
||||
)
|
||||
|
||||
|
||||
def reference_post_conv(
|
||||
@@ -207,3 +214,132 @@ def test_fused_post_conv_l0():
|
||||
)
|
||||
assert q.shape == (0, H, K)
|
||||
assert g.shape == (0, HV)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size,query_lengths,state_dtype,norm_dtype",
|
||||
[
|
||||
pytest.param(16, (4, 4), torch.bfloat16, torch.bfloat16, id="tp16-bf16"),
|
||||
pytest.param(4, (4, 4), torch.float32, torch.float32, id="tp4-fp32"),
|
||||
pytest.param(16, (4, 2, 0), torch.bfloat16, torch.float32, id="tp16-ragged"),
|
||||
pytest.param(4, (4, 2, 0), torch.float32, torch.bfloat16, id="tp4-ragged"),
|
||||
pytest.param(16, (8,), torch.float32, torch.bfloat16, id="tp16-max"),
|
||||
pytest.param(4, (8,), torch.bfloat16, torch.float32, id="tp4-max"),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_gdn_decode_post_conv_mtp_ratio8(
|
||||
tp_size: int,
|
||||
query_lengths: tuple[int, ...],
|
||||
state_dtype: torch.dtype,
|
||||
norm_dtype: torch.dtype,
|
||||
) -> None:
|
||||
if torch.cuda.get_device_capability() < (8, 0):
|
||||
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
|
||||
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
|
||||
pytest.skip("fused GDN decode MTP op is not built")
|
||||
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
H = 16 // tp_size
|
||||
HV = 128 // tp_size
|
||||
K = V = 128
|
||||
num_reqs = len(query_lengths)
|
||||
state_width = max(query_lengths)
|
||||
num_tokens = sum(query_lengths)
|
||||
num_slots = num_reqs * state_width + 1
|
||||
scale = K**-0.5
|
||||
eps = 1e-6
|
||||
|
||||
mixed_qkv = torch.randn(
|
||||
num_tokens,
|
||||
2 * H * K + HV * V,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
query, key, value = torch.split(
|
||||
mixed_qkv,
|
||||
[H * K, H * K, HV * V],
|
||||
dim=-1,
|
||||
)
|
||||
query = query.view(1, num_tokens, H, K)
|
||||
key = key.view(1, num_tokens, H, K)
|
||||
value = value.view(1, num_tokens, HV, V)
|
||||
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
|
||||
b, a = ba.chunk(2, dim=-1)
|
||||
assert not a.is_contiguous()
|
||||
assert not b.is_contiguous()
|
||||
A_log = 0.5 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
output_gate = torch.randn(num_tokens, HV, V, dtype=torch.bfloat16, device=device)
|
||||
norm_weight = torch.randn(V, dtype=norm_dtype, device=device)
|
||||
state_ref = (
|
||||
0.01 * torch.randn(num_slots, HV, V, K, dtype=torch.float32, device=device)
|
||||
).to(state_dtype)
|
||||
state_actual = state_ref.clone()
|
||||
state_indices = torch.arange(1, num_slots, dtype=torch.int32, device=device).view(
|
||||
num_reqs, state_width
|
||||
)
|
||||
cu_seqlens = torch.tensor(
|
||||
[0, *torch.tensor(query_lengths).cumsum(0).tolist()],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
num_accepted_tokens = torch.ones(num_reqs, dtype=torch.int32, device=device)
|
||||
if query_lengths[-1] == 0:
|
||||
state_indices[-1].zero_()
|
||||
|
||||
for step, accepted_tokens in enumerate((1, min(2, state_width), state_width)):
|
||||
num_accepted_tokens.fill_(accepted_tokens)
|
||||
if query_lengths[-1] == 0:
|
||||
num_accepted_tokens[-1] = 1
|
||||
raw_ref, _ = fused_sigmoid_gating_delta_rule_update(
|
||||
A_log=A_log,
|
||||
a=a,
|
||||
b=b,
|
||||
dt_bias=dt_bias,
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
initial_state=state_ref,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
scale=scale,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
expected = rmsnorm_fn(
|
||||
raw_ref.squeeze(0),
|
||||
norm_weight,
|
||||
None,
|
||||
z=output_gate,
|
||||
eps=eps,
|
||||
norm_before_gate=True,
|
||||
activation="silu",
|
||||
)
|
||||
actual = ops.fused_gdn_decode_post_conv_mtp(
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
state_indices=state_indices,
|
||||
cu_seqlens=cu_seqlens,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
state=state_actual,
|
||||
output_gate=output_gate,
|
||||
norm_weight=norm_weight,
|
||||
out=torch.empty_like(output_gate),
|
||||
scale=scale,
|
||||
norm_eps=eps,
|
||||
)
|
||||
|
||||
output_error = (actual.float() - expected.float()).norm()
|
||||
output_relative_l2 = output_error / expected.float().norm().clamp_min(1e-20)
|
||||
assert output_relative_l2 < 5e-4, (
|
||||
f"MTP output relative L2 mismatch at step {step}: "
|
||||
f"{output_relative_l2.item():.6g}"
|
||||
)
|
||||
|
||||
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
|
||||
|
||||
@@ -251,6 +251,21 @@ class TestEnvWithChoices:
|
||||
env_func()
|
||||
|
||||
|
||||
def test_gdn_decode_kernel_env(monkeypatch: pytest.MonkeyPatch):
|
||||
env_func = environment_variables["VLLM_GDN_DECODE_KERNEL"]
|
||||
monkeypatch.delenv("VLLM_GDN_DECODE_KERNEL", raising=False)
|
||||
assert env_func() == "cuda"
|
||||
|
||||
for value in ("cuda", "triton"):
|
||||
monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value)
|
||||
assert env_func() == value
|
||||
|
||||
for value in ("fused", "invalid"):
|
||||
monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value)
|
||||
with pytest.raises(ValueError, match="VLLM_GDN_DECODE_KERNEL"):
|
||||
env_func()
|
||||
|
||||
|
||||
class TestEnvListWithChoices:
|
||||
"""Test cases for env_list_with_choices function."""
|
||||
|
||||
|
||||
@@ -2777,6 +2777,43 @@ def fused_kda_decode(
|
||||
return out
|
||||
|
||||
|
||||
def fused_gdn_decode_post_conv_mtp(
|
||||
mixed_qkv: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
output_gate: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
scale: float = 128**-0.5,
|
||||
norm_eps: float = 1e-5,
|
||||
) -> torch.Tensor:
|
||||
if out is None:
|
||||
out = torch.empty_like(output_gate)
|
||||
torch.ops._C.fused_gdn_decode_post_conv_mtp(
|
||||
mixed_qkv,
|
||||
a,
|
||||
b,
|
||||
A_log,
|
||||
dt_bias,
|
||||
state_indices,
|
||||
cu_seqlens,
|
||||
num_accepted_tokens,
|
||||
state,
|
||||
output_gate,
|
||||
norm_weight,
|
||||
out,
|
||||
scale,
|
||||
norm_eps,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def concat_and_cache_mla(
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
|
||||
@@ -769,6 +769,7 @@ class CompilationConfig:
|
||||
"vllm::short_conv",
|
||||
"vllm::linear_attention",
|
||||
"vllm::qwen_gdn_attention_core",
|
||||
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
||||
"vllm::gdn_attention_core_xpu",
|
||||
"vllm::olmo_hybrid_gdn_full_forward",
|
||||
"vllm::sparse_attn_indexer",
|
||||
|
||||
@@ -127,6 +127,7 @@ if TYPE_CHECKING:
|
||||
VLLM_DISABLED_KERNELS: list[str] = []
|
||||
VLLM_USE_HW_AGNOSTIC: bool = False
|
||||
VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True
|
||||
VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda"
|
||||
VLLM_DISABLE_PYNCCL: bool = False
|
||||
VLLM_USE_OINK_OPS: bool = False
|
||||
VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD: bool = True
|
||||
@@ -1209,6 +1210,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1"))
|
||||
),
|
||||
# Select the GDN MTP decode implementation. "cuda" uses the fused decode
|
||||
# kernel where supported and falls back to "triton" otherwise; setting it
|
||||
# explicitly to "cuda" raises when unsupported.
|
||||
"VLLM_GDN_DECODE_KERNEL": env_with_choices(
|
||||
"VLLM_GDN_DECODE_KERNEL",
|
||||
"cuda",
|
||||
["cuda", "triton"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# Disable pynccl (using torch.distributed instead)
|
||||
"VLLM_DISABLE_PYNCCL": lambda: (
|
||||
os.getenv("VLLM_DISABLE_PYNCCL", "False").lower() in ("true", "1")
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen3-Next/Qwen3.5 model."""
|
||||
|
||||
import os
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import (
|
||||
@@ -84,6 +86,9 @@ if GDN_AITER_TRITON_AVAILABLE:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
MAX_FUSED_GDN_MTP_TOKENS = 8
|
||||
FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
|
||||
|
||||
|
||||
def _resolve_gdn_prefill_backend(
|
||||
vllm_config: VllmConfig,
|
||||
@@ -484,12 +489,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.enable_packed_recurrent_decode = (
|
||||
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
|
||||
)
|
||||
self.gdn_decode_kernel = envs.VLLM_GDN_DECODE_KERNEL.strip().lower()
|
||||
if self.gdn_decode_kernel == "cuda":
|
||||
reason = self._fused_gdn_decode_unsupported_reason(vllm_config)
|
||||
if reason is not None:
|
||||
if "VLLM_GDN_DECODE_KERNEL" in os.environ:
|
||||
raise ValueError(
|
||||
f"VLLM_GDN_DECODE_KERNEL=cuda is not supported: {reason}"
|
||||
)
|
||||
logger.info_once(
|
||||
"Falling back to the Triton GDN decode path: %s", reason
|
||||
)
|
||||
self.gdn_decode_kernel = "triton"
|
||||
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
|
||||
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def _fused_gdn_decode_unsupported_reason(
|
||||
self, vllm_config: VllmConfig
|
||||
) -> str | None:
|
||||
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()
|
||||
if (
|
||||
self.gqa_interleaved_layout
|
||||
or self.head_k_dim != 128
|
||||
or self.head_v_dim != 128
|
||||
or self.norm.activation != "silu"
|
||||
or vllm_config.model_config.dtype != torch.bfloat16
|
||||
or conv_state_dtype != torch.bfloat16
|
||||
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
|
||||
or not current_platform.has_device_capability(80)
|
||||
):
|
||||
return (
|
||||
"the fused CUDA kernel requires a BF16 GDN model with "
|
||||
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
|
||||
"convolution cache, BF16 or FP32 recurrent state, and a "
|
||||
"GPU with compute capability 8.0+"
|
||||
)
|
||||
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
|
||||
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
|
||||
return None
|
||||
|
||||
def create_qkvz_proj(
|
||||
self,
|
||||
hidden_size: int,
|
||||
@@ -846,6 +889,26 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
ba, _ = self.in_proj_ba(hidden_states)
|
||||
|
||||
use_fused_gdn_decode = (
|
||||
self.enable_fused_gdn_decode
|
||||
and hidden_states.dtype == torch.bfloat16
|
||||
and self.norm.weight.dtype in (torch.bfloat16, torch.float32)
|
||||
)
|
||||
if use_fused_gdn_decode:
|
||||
core_attn_out = torch.zeros(
|
||||
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed(
|
||||
mixed_qkvz,
|
||||
ba,
|
||||
core_attn_out,
|
||||
layer_name=_encode_layer_name(self.prefix),
|
||||
)
|
||||
output, _ = self.out_proj(core_attn_out.flatten(-2))
|
||||
return output
|
||||
|
||||
if self.gqa_interleaved_layout:
|
||||
# Qwen3-Next: unpack the interleaved GQA layout
|
||||
query, key, value, z, b, a = self.fix_query_key_value_ordering(
|
||||
@@ -862,8 +925,6 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
mixed_qkv, z = mixed_qkvz.split([qkv_size, z_size], dim=-1)
|
||||
z = z.reshape(z.size(0), -1, self.head_v_dim)
|
||||
b, a = self.split_ba(ba)
|
||||
b = b.contiguous()
|
||||
a = a.contiguous()
|
||||
|
||||
# ============================================================
|
||||
# Part 2: Core Attention (Custom Op)
|
||||
@@ -878,8 +939,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
|
||||
torch.ops.vllm.qwen_gdn_attention_core(
|
||||
mixed_qkv,
|
||||
b,
|
||||
a,
|
||||
b.contiguous(),
|
||||
a.contiguous(),
|
||||
core_attn_out,
|
||||
layer_name=_encode_layer_name(self.prefix),
|
||||
)
|
||||
@@ -1622,6 +1683,211 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
)
|
||||
return
|
||||
|
||||
def _forward_core_decode_spec_fused_norm(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
output_gate: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
attn_metadata: GDNAttentionMetadata,
|
||||
) -> None:
|
||||
state_indices = attn_metadata.spec_state_indices_tensor
|
||||
cu_seqlens = attn_metadata.spec_query_start_loc
|
||||
num_accepted_tokens = attn_metadata.num_accepted_tokens
|
||||
assert state_indices is not None
|
||||
assert cu_seqlens is not None
|
||||
assert num_accepted_tokens is not None
|
||||
|
||||
num_requests = attn_metadata.num_spec_decodes
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
conv_state = (
|
||||
self.kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self.kv_cache[0].transpose(-1, -2)
|
||||
)
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
mixed_qkv = causal_conv1d_update(
|
||||
mixed_qkv[:num_actual_tokens],
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
self.activation,
|
||||
conv_state_indices=state_indices[:num_requests, 0],
|
||||
num_accepted_tokens=num_accepted_tokens[:num_requests],
|
||||
query_start_loc=cu_seqlens[: num_requests + 1],
|
||||
max_query_len=state_indices.size(1),
|
||||
validate_data=False,
|
||||
)
|
||||
self._forward_core_decode_spec_post_conv_fused_norm(
|
||||
mixed_qkv=mixed_qkv,
|
||||
b=b[:num_actual_tokens],
|
||||
a=a[:num_actual_tokens],
|
||||
output_gate=output_gate[:num_actual_tokens],
|
||||
core_attn_out=core_attn_out[:num_actual_tokens],
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
|
||||
def _forward_core_decode_spec_post_conv_fused_norm(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
output_gate: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
attn_metadata: GDNAttentionMetadata,
|
||||
) -> None:
|
||||
state_indices = attn_metadata.spec_state_indices_tensor
|
||||
cu_seqlens = attn_metadata.spec_query_start_loc
|
||||
num_accepted_tokens = attn_metadata.num_accepted_tokens
|
||||
assert state_indices is not None
|
||||
assert cu_seqlens is not None
|
||||
assert num_accepted_tokens is not None
|
||||
|
||||
num_requests = attn_metadata.num_spec_decodes
|
||||
ops.fused_gdn_decode_post_conv_mtp(
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
state_indices=state_indices[:num_requests],
|
||||
cu_seqlens=cu_seqlens[: num_requests + 1],
|
||||
num_accepted_tokens=num_accepted_tokens[:num_requests],
|
||||
state=self.kv_cache[1],
|
||||
output_gate=output_gate,
|
||||
norm_weight=self.norm.weight,
|
||||
out=core_attn_out,
|
||||
scale=self.head_k_dim**-0.5,
|
||||
norm_eps=self.layer_norm_epsilon,
|
||||
)
|
||||
|
||||
def _forward_core_fused_norm_packed(
|
||||
self,
|
||||
mixed_qkvz: torch.Tensor,
|
||||
ba: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
|
||||
if attn_metadata_raw is None:
|
||||
self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0)
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
mixed_qkv, output_gate_flat = mixed_qkvz.split(
|
||||
[qkv_size, self.value_dim // self.tp_size], dim=-1
|
||||
)
|
||||
output_gate = output_gate_flat.reshape(
|
||||
output_gate_flat.size(0), -1, self.head_v_dim
|
||||
)
|
||||
b, a = self.split_ba(ba)
|
||||
self._forward_core_fused_norm(
|
||||
mixed_qkv=mixed_qkv,
|
||||
b=b,
|
||||
a=a,
|
||||
output_gate=output_gate,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
|
||||
def _can_use_fused_gdn_mtp_decode(
|
||||
self, attn_metadata: GDNAttentionMetadata
|
||||
) -> bool:
|
||||
state_indices = attn_metadata.spec_state_indices_tensor
|
||||
return (
|
||||
attn_metadata.spec_sequence_masks is not None
|
||||
and attn_metadata.num_decodes == 0
|
||||
and attn_metadata.num_spec_decodes > 0
|
||||
and self.kv_cache[1].dtype in FUSED_GDN_STATE_DTYPES
|
||||
and self.gdn_decode_kernel == "cuda"
|
||||
and self.num_v_heads == 8 * self.num_k_heads
|
||||
and state_indices is not None
|
||||
and state_indices.size(1) <= MAX_FUSED_GDN_MTP_TOKENS
|
||||
and hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp")
|
||||
)
|
||||
|
||||
def _rms_norm_gated_cuda(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
output_gate: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
layer_norm_fwd,
|
||||
)
|
||||
|
||||
x_shape = x.shape
|
||||
assert output_gate.shape == x_shape
|
||||
assert out.shape == x_shape
|
||||
x_2d = x.reshape(-1, x_shape[-1])
|
||||
output_gate_2d = output_gate.reshape(-1, x_shape[-1])
|
||||
out_2d = out.reshape(-1, x_shape[-1])
|
||||
assert x_2d.stride(-1) == 1
|
||||
assert output_gate_2d.stride(-1) == 1
|
||||
assert out_2d.stride(-1) == 1
|
||||
layer_norm_fwd(
|
||||
x_2d,
|
||||
self.norm.weight.contiguous(),
|
||||
self.norm.bias,
|
||||
self.norm.eps,
|
||||
z=output_gate_2d,
|
||||
out=out_2d,
|
||||
group_size=(
|
||||
x_shape[-1] if self.norm.group_size is None else self.norm.group_size
|
||||
),
|
||||
norm_before_gate=self.norm.norm_before_gate,
|
||||
is_rms_norm=True,
|
||||
activation=self.norm.activation,
|
||||
)
|
||||
|
||||
def _forward_core_fused_norm(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
output_gate: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
if attn_metadata_raw is None:
|
||||
self._warmup_prefill_kernels(mixed_qkv, 0)
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
if (
|
||||
self._can_use_fused_gdn_mtp_decode(attn_metadata)
|
||||
and attn_metadata.num_prefills == 0
|
||||
):
|
||||
self._forward_core_decode_spec_fused_norm(
|
||||
mixed_qkv=mixed_qkv,
|
||||
b=b,
|
||||
a=a,
|
||||
output_gate=output_gate,
|
||||
core_attn_out=core_attn_out,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
return
|
||||
self._forward_core(
|
||||
mixed_qkv=mixed_qkv,
|
||||
b=b.contiguous(),
|
||||
a=a.contiguous(),
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
self._rms_norm_gated_cuda(
|
||||
core_attn_out[:num_actual_tokens],
|
||||
output_gate[:num_actual_tokens],
|
||||
core_attn_out[:num_actual_tokens],
|
||||
)
|
||||
|
||||
|
||||
def qwen_gdn_attention_core(
|
||||
qkv_or_qkvz: torch.Tensor,
|
||||
@@ -1683,6 +1949,39 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def qwen_gdn_attention_core_fused_norm_packed(
|
||||
mixed_qkvz: torch.Tensor,
|
||||
ba: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._forward_core_fused_norm_packed(
|
||||
mixed_qkvz=mixed_qkvz,
|
||||
ba=ba,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
|
||||
|
||||
def gdn_attention_core_fused_norm_packed_fake(
|
||||
mixed_qkvz: torch.Tensor,
|
||||
ba: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen_gdn_attention_core_fused_norm_packed",
|
||||
op_func=qwen_gdn_attention_core_fused_norm_packed,
|
||||
mutates_args=["core_attn_out"],
|
||||
fake_impl=gdn_attention_core_fused_norm_packed_fake,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fused_gdn_gating_kernel(
|
||||
g,
|
||||
|
||||
Reference in New Issue
Block a user