[Kimi-K3] Add AttnRes kernels (#50090)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-07-28 00:22:09 -07:00
committed by GitHub
parent 99b57a4823
commit 61ac368021
14 changed files with 1719 additions and 0 deletions
+12
View File
@@ -61,6 +61,18 @@ steps:
# FA4 kernel tests require SM100; the suite skips them elsewhere.
- pytest -v -s models/inkling
- label: Kimi K3 Unit Tests (B200)
key: kimi-k3-unit-tests-b200
timeout_in_minutes: 40
device: b200-k8s
source_file_dependencies:
- vllm/models/kimi_k3/
- csrc/libtorch_stable/kimi_k3/
- tests/models/kimi_k3/
commands:
# The native NVIDIA AttnRes kernel requires the SM100 family.
- pytest -v -s models/kimi_k3
- label: Basic Models Test (Other CPU) # 5min
key: basic-models-test-other-cpu
depends_on:
+22
View File
@@ -1079,6 +1079,24 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
set(MLA_ARCHS)
endif()
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
cuda_archs_loose_intersection(KIMI_K3_ATTN_RES_ARCHS
"10.0f" "${CUDA_ARCHS}")
endif()
if(KIMI_K3_ATTN_RES_ARCHS)
set(KIMI_K3_ATTN_RES_SRC
"csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu")
set_gencode_flags_for_srcs(
SRCS "${KIMI_K3_ATTN_RES_SRC}"
CUDA_ARCHS "${KIMI_K3_ATTN_RES_ARCHS}")
set_property(SOURCE ${KIMI_K3_ATTN_RES_SRC} APPEND PROPERTY
COMPILE_OPTIONS
"$<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr;--expt-extended-lambda;--use_fast_math>")
list(APPEND VLLM_STABLE_EXT_SRC "${KIMI_K3_ATTN_RES_SRC}")
message(STATUS
"Building Kimi K3 AttnRes for archs: ${KIMI_K3_ATTN_RES_ARCHS}")
endif()
# Hadacore kernels
cuda_archs_loose_intersection(HADACORE_ARCHS "8.0+PTX;9.0+PTX" "${CUDA_ARCHS}")
if(HADACORE_ARCHS)
@@ -1120,6 +1138,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
target_compile_definitions(_C_stable_libtorch PRIVATE
VLLM_ENABLE_COOPERATIVE_TOPK=1)
endif()
if(KIMI_K3_ATTN_RES_ARCHS)
target_compile_definitions(_C_stable_libtorch PRIVATE
VLLM_ENABLE_KIMI_K3_ATTN_RES=1)
endif()
# Needed by CUTLASS kernels
target_compile_definitions(_C_stable_libtorch PRIVATE
CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
@@ -0,0 +1,954 @@
/*
* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
*/
// Production AttnRes forward for Blackwell (SM100).
//
// Warp-specialized online softmax + residual + RMSNorm:
// - 1 producer warp issues cp.async.bulk row loads into shared memory.
// - 8 consumer warps compute reductions and output.
// - Q=res_weight*rms_weight remains in registers across persistent tokens.
// - V rows are converted once and cached as FP32 in TMEM between passes.
//
// Integration contract: Kimi K3 H=7168, 1<=num_blocks<=8, and token-major
// block residual storage.
#include "../torch_utils.h"
#include <cfloat>
#include <cstdint>
#include <cstdio>
#include <cuda_runtime.h>
#include <type_traits>
using bf16_t = __nv_bfloat16;
namespace sm100 {
namespace fwd_prod_v2 {
constexpr int K_TILE = 1024;
constexpr int N_CHUNK_DEFAULT = 4;
constexpr int CHUNK_DEPTH = 2;
constexpr int BLK = 288; // 1 producer warp + 8 consumer warps
constexpr int CONSUMER_THREADS = BLK - 32; // 256
constexpr int CONSUMER_WARPS = CONSUMER_THREADS / 32;
constexpr int CONSUMER_GROUPS = 2; // two 128-thread consumer groups
constexpr int CONSUMER_THREADS_PER_GROUP = CONSUMER_THREADS / CONSUMER_GROUPS;
constexpr int FIRST_USER_NAMED_BARRIER = 8;
__device__ __forceinline__ const bf16_t* residual_addr(
const bf16_t* block_res, const bf16_t* layer_res, int source, int N,
int token, int block_stride_m, int block_stride_r, int H) {
if (source < N - 1) {
return block_res + static_cast<long long>(token) * block_stride_m +
source * block_stride_r;
}
return layer_res + static_cast<long long>(token) * H;
}
__device__ __forceinline__ uint32_t elect_one_sync() {
uint32_t pred = 0;
uint32_t laneid = 0;
asm volatile(
"{\n"
".reg .b32 %%rx;\n"
".reg .pred %%px;\n"
" elect.sync %%rx|%%px, %2;\n"
"@%%px mov.s32 %1, 1;\n"
" mov.s32 %0, %%rx;\n"
"}\n"
: "+r"(laneid), "+r"(pred)
: "r"(0xffffffff));
return pred;
}
__device__ __forceinline__ void mbarrier_init(uint64_t& barrier,
int thread_count) {
uint32_t const barrier_addr =
static_cast<uint32_t>(__cvta_generic_to_shared(&barrier));
asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" ::"r"(barrier_addr),
"r"(thread_count));
}
__device__ __forceinline__ void mbarrier_expect_tx(uint64_t& barrier,
uint32_t bytes) {
uint32_t const barrier_addr =
static_cast<uint32_t>(__cvta_generic_to_shared(&barrier));
asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" ::"r"(
barrier_addr),
"r"(bytes));
}
__device__ __forceinline__ void mbarrier_wait(uint64_t& barrier, int phase) {
uint32_t const barrier_addr =
static_cast<uint32_t>(__cvta_generic_to_shared(&barrier));
asm volatile(
"{\n"
".reg .pred p;\n"
"WAIT:\n"
"mbarrier.try_wait.parity.shared::cta.b64 p, [%0], %1;\n"
"@p bra DONE;\n"
"bra WAIT;\n"
"DONE:\n"
"}\n" ::"r"(barrier_addr),
"r"(phase));
}
__device__ __forceinline__ void mbarrier_arrive(uint64_t& barrier) {
uint32_t const barrier_addr =
static_cast<uint32_t>(__cvta_generic_to_shared(&barrier));
asm volatile(
"{\n"
".reg .b64 state;\n"
"mbarrier.arrive.shared::cta.b64 state, [%0];\n"
"}\n" ::"r"(barrier_addr));
}
__device__ __forceinline__ void fence_mbarrier_init() {
asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory");
}
__device__ __forceinline__ void named_barrier_sync(uint32_t num_threads,
uint32_t user_barrier_id) {
asm volatile(
"bar.sync %0, %1;" ::"r"(user_barrier_id + FIRST_USER_NAMED_BARRIER),
"r"(num_threads)
: "memory");
}
__device__ __forceinline__ void tmem_allocate(int num_columns, uint32_t* dst) {
uint32_t const dst_addr =
static_cast<uint32_t>(__cvta_generic_to_shared(dst));
asm volatile(
"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"(
dst_addr),
"r"(num_columns));
}
__device__ __forceinline__ void tmem_free(uint32_t tmem_ptr, int num_columns) {
asm volatile(
"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" ::"r"(tmem_ptr),
"r"(num_columns));
}
__device__ __forceinline__ void tmem_release_allocation_lock() {
asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;");
}
__device__ __forceinline__ void tmem_store_wait() {
asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory");
}
template <int N, typename T>
__device__ __forceinline__ void tmem_load(uint32_t src_addr, T* dst) {
uint32_t* values = reinterpret_cast<uint32_t*>(dst);
if constexpr (N == 8) {
asm volatile(
"tcgen05.ld.sync.aligned.32x32b.x8.b32"
"{%0, %1, %2, %3, %4, %5, %6, %7}, [%8];\n"
: "=r"(values[0]), "=r"(values[1]), "=r"(values[2]), "=r"(values[3]),
"=r"(values[4]), "=r"(values[5]), "=r"(values[6]), "=r"(values[7])
: "r"(src_addr));
} else {
static_assert(N == 4, "AttnRes TMEM helpers support x4 and x8");
asm volatile(
"tcgen05.ld.sync.aligned.32x32b.x4.b32"
"{%0, %1, %2, %3}, [%4];\n"
: "=r"(values[0]), "=r"(values[1]), "=r"(values[2]), "=r"(values[3])
: "r"(src_addr));
}
}
template <int N, typename T>
__device__ __forceinline__ void tmem_store(uint32_t dst_addr, T* src) {
uint32_t* values = reinterpret_cast<uint32_t*>(src);
if constexpr (N == 8) {
asm volatile(
"tcgen05.st.sync.aligned.32x32b.x8.b32"
"[%8], {%0, %1, %2, %3, %4, %5, %6, %7};\n" ::"r"(values[0]),
"r"(values[1]), "r"(values[2]), "r"(values[3]), "r"(values[4]),
"r"(values[5]), "r"(values[6]), "r"(values[7]), "r"(dst_addr));
} else {
static_assert(N == 4, "AttnRes TMEM helpers support x4 and x8");
asm volatile(
"tcgen05.st.sync.aligned.32x32b.x4.b32"
"[%4], {%0, %1, %2, %3};\n" ::"r"(values[0]),
"r"(values[1]), "r"(values[2]), "r"(values[3]), "r"(dst_addr));
}
}
__device__ __forceinline__ float2 float2_add(const float2& a, const float2& b) {
float2 result;
asm volatile("add.rn.f32x2 %0, %1, %2;\n"
: "=l"(reinterpret_cast<uint64_t&>(result))
: "l"(reinterpret_cast<uint64_t const&>(a)),
"l"(reinterpret_cast<uint64_t const&>(b)));
return result;
}
__device__ __forceinline__ float2 float2_mul(const float2& a, const float2& b) {
float2 result;
asm volatile("mul.f32x2 %0, %1, %2;\n"
: "=l"(reinterpret_cast<uint64_t&>(result))
: "l"(reinterpret_cast<uint64_t const&>(a)),
"l"(reinterpret_cast<uint64_t const&>(b)));
return result;
}
__device__ __forceinline__ float2 float2_fma(const float2& a, const float2& b,
const float2& c) {
float2 result;
asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n"
: "=l"(reinterpret_cast<uint64_t&>(result))
: "l"(reinterpret_cast<uint64_t const&>(a)),
"l"(reinterpret_cast<uint64_t const&>(b)),
"l"(reinterpret_cast<uint64_t const&>(c)));
return result;
}
template <int NC>
struct FwdSmemPlan {
alignas(16) uint64_t bar_ready[CHUNK_DEPTH];
alignas(16) uint64_t bar_consumed[CHUNK_DEPTH];
alignas(16) uint64_t bar_output_norm_ready;
alignas(16) float2 ws_stats[CONSUMER_WARPS][NC];
uint32_t tmem_base;
};
__device__ __forceinline__ void cp_async_bulk(void* smem_dst,
const void* gmem_src, int bytes,
uint64_t& mbar) {
uint32_t const s = static_cast<uint32_t>(__cvta_generic_to_shared(smem_dst));
uint32_t const m = static_cast<uint32_t>(__cvta_generic_to_shared(&mbar));
asm volatile(
"cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [%0], "
"[%1], %2, [%3];\n" ::"r"(s),
"l"(gmem_src), "r"(bytes), "r"(m)
: "memory");
}
template <int H, int NC = N_CHUNK_DEFAULT, bool RELEASE_TMEM = false,
bool HAS_DELTA = false, bool HAS_OUTPUT_NORM = false,
bool OUTPUT_NORM_IN_SMEM = false>
__global__ void __launch_bounds__(BLK, 1) attn_res_fwd_online_v2_kernel(
const bf16_t* __restrict__ block_res, bf16_t* __restrict__ layer_res,
const bf16_t* __restrict__ delta, const bf16_t* __restrict__ res_w,
const bf16_t* __restrict__ rms_w, bf16_t* __restrict__ output, int N, int T,
int B, int block_stride_m, int block_stride_r, float rms_eps,
const bf16_t* __restrict__ output_norm_weight, float output_norm_eps) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && __CUDA_ARCH__ < 1100
constexpr float LOG2_E = 1.4426950408889634f;
constexpr int N_CHUNK = NC;
// The two-source specialization only consumes half of the TMEM columns.
constexpr int TMEM_COLS_ALLOC = NC == 2 ? 128 : 256;
constexpr int NUM_BUFS = CHUNK_DEPTH * NC;
constexpr int NHT = H / K_TILE;
constexpr int SLICES_PER_GROUP =
(NHT + CONSUMER_GROUPS - 1) / CONSUMER_GROUPS;
constexpr int VEC = 8;
constexpr int ACC_PER_THREAD = H == 7168 ? 28 : SLICES_PER_GROUP * VEC;
constexpr int TMEM_V_COLS_PER_GROUP = SLICES_PER_GROUP * N_CHUNK * VEC;
constexpr int TMEM_V_COLS_TOTAL = CONSUMER_GROUPS * TMEM_V_COLS_PER_GROUP;
static_assert(TMEM_V_COLS_TOTAL <= TMEM_COLS_ALLOC);
static_assert(H >= 4096 && H <= 8192);
static_assert(H % K_TILE == 0);
const int tid = threadIdx.x;
const int wid = tid >> 5;
const int lane = tid & 31;
const int TB = T * B;
const int num_ctas = gridDim.x;
const int num_chunks = (N + N_CHUNK - 1) / N_CHUNK;
const int comp_wid = wid - 1;
const int comp_tid = tid - 32;
const int group = (comp_wid >= 4) ? 1 : 0;
const int ct_in_group =
(comp_tid >= 0) ? (comp_tid & (CONSUMER_THREADS_PER_GROUP - 1)) : -1;
const int k_local = ct_in_group * VEC;
constexpr size_t V_BYTES = (size_t)NUM_BUFS * H * sizeof(bf16_t);
constexpr size_t DELTA_BYTES =
HAS_DELTA ? (size_t)CHUNK_DEPTH * H * sizeof(bf16_t) : 0;
constexpr size_t OUTPUT_NORM_BYTES =
OUTPUT_NORM_IN_SMEM ? (size_t)H * sizeof(bf16_t) : 0;
extern __shared__ __align__(16) char smem_raw[];
bf16_t* v_bufs = reinterpret_cast<bf16_t*>(smem_raw); // [NUM_BUFS][H]
bf16_t* delta_bufs = reinterpret_cast<bf16_t*>(smem_raw + V_BYTES);
bf16_t* output_norm_buf =
reinterpret_cast<bf16_t*>(smem_raw + V_BYTES + DELTA_BYTES);
FwdSmemPlan<NC>& plan = *reinterpret_cast<FwdSmemPlan<NC>*>(
smem_raw + V_BYTES + DELTA_BYTES + OUTPUT_NORM_BYTES);
auto slot_of = [](long long gci, int n) {
return (int)(gci % CHUNK_DEPTH) * N_CHUNK + n;
};
auto phase_of = [](long long gci) { return (int)((gci / CHUNK_DEPTH) & 1); };
auto buf_ptr = [&](int slot) -> bf16_t* { return v_bufs + slot * H; };
auto delta_buf_ptr = [&](int chunk_slot) -> bf16_t* {
return delta_bufs + chunk_slot * H;
};
if (wid == 0 && elect_one_sync()) {
#pragma unroll
for (int i = 0; i < CHUNK_DEPTH; i++) {
mbarrier_init(plan.bar_ready[i], 1);
mbarrier_init(plan.bar_consumed[i], CONSUMER_WARPS);
}
if constexpr (OUTPUT_NORM_IN_SMEM) {
mbarrier_init(plan.bar_output_norm_ready, 1);
}
fence_mbarrier_init();
}
// gdc wait BEFORE tmem alloc
cudaGridDependencySynchronize();
if (wid == 1) {
tmem_allocate(TMEM_COLS_ALLOC, &plan.tmem_base);
if constexpr (RELEASE_TMEM) {
tmem_release_allocation_lock();
}
}
__syncthreads();
if constexpr (OUTPUT_NORM_IN_SMEM) {
if (wid == 0 && elect_one_sync()) {
mbarrier_expect_tx(plan.bar_output_norm_ready, H * (int)sizeof(bf16_t));
cp_async_bulk(output_norm_buf, output_norm_weight, H * sizeof(bf16_t),
plan.bar_output_norm_ready);
}
}
const uint32_t my_v_tmem =
comp_tid >= 0 ? plan.tmem_base + group * TMEM_V_COLS_PER_GROUP : 0;
float q_cache[ACC_PER_THREAD];
if (comp_tid >= 0) {
#pragma unroll
for (int si = 0; si < SLICES_PER_GROUP; si++) {
if constexpr (H == 7168) {
if (si == SLICES_PER_GROUP - 1) {
int h_base = 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4;
#pragma unroll
for (int j = 0; j < 4; j++) {
int h = h_base + j;
q_cache[si * VEC + j] =
__bfloat162float(rms_w[h]) * __bfloat162float(res_w[h]);
}
continue;
}
}
int dt = si * CONSUMER_GROUPS + group;
if (dt >= NHT) continue;
int h_base = dt * K_TILE + k_local;
#pragma unroll
for (int j = 0; j < VEC; j++) {
int h = h_base + j;
q_cache[si * VEC + j] =
__bfloat162float(rms_w[h]) * __bfloat162float(res_w[h]);
}
}
}
if (wid == 0) {
if (elect_one_sync()) {
long long gci = 0;
for (int tb = blockIdx.x; tb < TB; tb += num_ctas) {
const int t = tb / B;
for (int ci = 0; ci < num_chunks; ci++, gci++) {
int ns = ci * N_CHUNK;
int an = min(N_CHUNK, N - ns);
int chunk_slot = (int)(gci % CHUNK_DEPTH);
int pc = phase_of(gci);
mbarrier_wait(plan.bar_consumed[chunk_slot], pc ^ 1);
int transaction_bytes = an * H * (int)sizeof(bf16_t);
if constexpr (HAS_DELTA) {
int prefix_n = N - 1 - ns;
if (prefix_n >= 0 && prefix_n < an) {
transaction_bytes += H * (int)sizeof(bf16_t);
}
}
mbarrier_expect_tx(plan.bar_ready[chunk_slot], transaction_bytes);
#pragma unroll
for (int n = 0; n < N_CHUNK; n++) {
if (n >= an) continue;
int slot = slot_of(gci, n);
const bf16_t* src =
residual_addr(block_res, layer_res, ns + n, N, t,
block_stride_m, block_stride_r, H);
cp_async_bulk(buf_ptr(slot), src, H * sizeof(bf16_t),
plan.bar_ready[chunk_slot]);
}
if constexpr (HAS_DELTA) {
int prefix_n = N - 1 - ns;
if (prefix_n >= 0 && prefix_n < an) {
cp_async_bulk(delta_buf_ptr(chunk_slot),
delta + (long long)tb * H, H * sizeof(bf16_t),
plan.bar_ready[chunk_slot]);
}
}
}
}
}
} else {
float acc32[ACC_PER_THREAD] = {};
float eps_cache;
asm volatile("mov.b32 %0, %1;" : "=f"(eps_cache) : "f"(rms_eps));
long long gci = 0;
for (int tb = blockIdx.x; tb < TB; tb += num_ctas) {
float m_running = -FLT_MAX;
float s_running = 0.f;
#pragma unroll
for (int i = 0; i < ACC_PER_THREAD; i++) {
acc32[i] = 0.f;
}
for (int ci = 0; ci < num_chunks; ci++, gci++) {
int ns = ci * N_CHUNK;
int an = min(N_CHUNK, N - ns);
int chunk_slot = (int)(gci % CHUNK_DEPTH);
int pr = phase_of(gci);
mbarrier_wait(plan.bar_ready[chunk_slot], pr);
float2 sq_local[N_CHUNK] = {};
float2 dot_local[N_CHUNK] = {};
auto pass_A_body = [&](auto AN_TOK) {
constexpr int AN = decltype(AN_TOK)::value;
#pragma unroll
for (int si = 0; si < SLICES_PER_GROUP; si++) {
if constexpr (H == 7168) {
if (si == SLICES_PER_GROUP - 1) {
int h_base =
6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4;
const float* qv = &q_cache[si * VEC];
#pragma unroll
for (int n = 0; n < AN; n++) {
int slot = slot_of(gci, n);
int2 vp =
*reinterpret_cast<const int2*>(buf_ptr(slot) + h_base);
auto* v2 = reinterpret_cast<__nv_bfloat162*>(&vp);
if constexpr (HAS_DELTA) {
int prefix_n = N - 1 - ns;
if (n == prefix_n) {
const bf16_t* delta_ptr =
delta_buf_ptr(chunk_slot) + h_base;
#pragma unroll
for (int j = 0; j < 2; j++) {
auto delta2 = *reinterpret_cast<const __nv_bfloat162*>(
delta_ptr + 2 * j);
v2[j] = __hadd2(v2[j], delta2);
}
*reinterpret_cast<int2*>(layer_res + (long long)tb * H +
h_base) = vp;
}
}
float2 f[2] = {__bfloat1622float2(v2[0]),
__bfloat1622float2(v2[1])};
tmem_store<4>(my_v_tmem + (si * N_CHUNK + n) * VEC, f);
sq_local[n] = float2_fma(f[0], f[0], sq_local[n]);
sq_local[n] = float2_fma(f[1], f[1], sq_local[n]);
dot_local[n] =
float2_fma(f[0], make_float2(qv[0], qv[1]), dot_local[n]);
dot_local[n] =
float2_fma(f[1], make_float2(qv[2], qv[3]), dot_local[n]);
}
continue;
}
}
int dt = si * CONSUMER_GROUPS + group;
if (dt >= NHT) continue;
int h_base = dt * K_TILE + k_local;
const float* qv = &q_cache[si * VEC];
#pragma unroll
for (int n = 0; n < AN; n++) {
int slot = slot_of(gci, n);
int4 vp = *reinterpret_cast<const int4*>(buf_ptr(slot) + h_base);
auto* v2 = reinterpret_cast<__nv_bfloat162*>(&vp);
if constexpr (HAS_DELTA) {
int prefix_n = N - 1 - ns;
if (n == prefix_n) {
const bf16_t* delta_ptr = delta_buf_ptr(chunk_slot) + h_base;
#pragma unroll
for (int j = 0; j < VEC / 2; j++) {
auto delta2 = *reinterpret_cast<const __nv_bfloat162*>(
delta_ptr + 2 * j);
v2[j] = __hadd2(v2[j], delta2);
}
*reinterpret_cast<int4*>(layer_res + (long long)tb * H +
h_base) = vp;
}
}
float2 f[4] = {
__bfloat1622float2(v2[0]), __bfloat1622float2(v2[1]),
__bfloat1622float2(v2[2]), __bfloat1622float2(v2[3])};
tmem_store<VEC>(my_v_tmem + (si * N_CHUNK + n) * VEC, f);
#pragma unroll
for (int j = 0; j < VEC / 2; j++) {
sq_local[n] = float2_fma(f[j], f[j], sq_local[n]);
dot_local[n] = float2_fma(
f[j], make_float2(qv[2 * j], qv[2 * j + 1]), dot_local[n]);
}
}
}
};
if constexpr (NC == 4) {
switch (an) {
case 4:
pass_A_body(std::integral_constant<int, 4>{});
break;
case 3:
pass_A_body(std::integral_constant<int, 3>{});
break;
case 2:
pass_A_body(std::integral_constant<int, 2>{});
break;
case 1:
pass_A_body(std::integral_constant<int, 1>{});
break;
default:
__builtin_unreachable();
}
} else if constexpr (NC == 3) {
switch (an) {
case 3:
pass_A_body(std::integral_constant<int, 3>{});
break;
case 2:
pass_A_body(std::integral_constant<int, 2>{});
break;
case 1:
pass_A_body(std::integral_constant<int, 1>{});
break;
default:
__builtin_unreachable();
}
} else {
static_assert(NC == 2);
switch (an) {
case 2:
pass_A_body(std::integral_constant<int, 2>{});
break;
case 1:
pass_A_body(std::integral_constant<int, 1>{});
break;
default:
__builtin_unreachable();
}
}
if (lane == 0) {
mbarrier_arrive(plan.bar_consumed[chunk_slot]);
}
tmem_store_wait();
float2 reduce_pair[N_CHUNK];
#pragma unroll
for (int n = 0; n < N_CHUNK; n++) {
reduce_pair[n] = make_float2(sq_local[n].x + sq_local[n].y,
dot_local[n].x + dot_local[n].y);
}
#pragma unroll
for (int offset = 16; offset > 0; offset >>= 1) {
#pragma unroll
for (int n = 0; n < N_CHUNK; n++) {
uint64_t packed = reinterpret_cast<uint64_t&>(reduce_pair[n]);
packed = __shfl_xor_sync(0xffffffff, packed, offset);
float2 other = reinterpret_cast<float2&>(packed);
reduce_pair[n] = float2_add(reduce_pair[n], other);
}
}
if (lane == 0) {
#pragma unroll
for (int n = 0; n < N_CHUNK; n++) {
plan.ws_stats[comp_wid][n] = reduce_pair[n];
}
}
named_barrier_sync(CONSUMER_THREADS, 0);
float local_rsig = 0.f;
float local_logit = 0.f;
int stat_n = lane / CONSUMER_WARPS;
int stat_w = lane % CONSUMER_WARPS;
float2 totals = {};
if (stat_n < N_CHUNK) {
totals = plan.ws_stats[stat_w][stat_n];
}
#pragma unroll
for (int offset = CONSUMER_WARPS / 2; offset > 0; offset >>= 1) {
totals.x +=
__shfl_down_sync(0xffffffff, totals.x, offset, CONSUMER_WARPS);
totals.y +=
__shfl_down_sync(0xffffffff, totals.y, offset, CONSUMER_WARPS);
}
if (stat_n < N_CHUNK && stat_w == 0) {
local_rsig = rsqrtf(totals.x / H + eps_cache);
local_logit = totals.y * local_rsig;
}
float logit_n[N_CHUNK];
#pragma unroll
for (int n = 0; n < N_CHUNK; n++) {
logit_n[n] = __shfl_sync(0xffffffff, local_logit, n * CONSUMER_WARPS);
}
float m_chunk = -FLT_MAX;
#pragma unroll
for (int n = 0; n < N_CHUNK; n++) {
if (n < an) m_chunk = fmaxf(m_chunk, logit_n[n]);
}
float m_new = fmaxf(m_running, m_chunk);
float corr = exp2f((m_running - m_new) * LOG2_E);
float w_n[N_CHUNK] = {};
float w_sum = 0.f;
#pragma unroll
for (int n = 0; n < N_CHUNK; n++) {
if (n < an) {
w_n[n] = exp2f((logit_n[n] - m_new) * LOG2_E);
w_sum += w_n[n];
}
}
auto pass_B_body = [&](auto AN_TOK) {
constexpr int AN = decltype(AN_TOK)::value;
#pragma unroll
for (int si = 0; si < SLICES_PER_GROUP; si++) {
if constexpr (H == 7168) {
if (si == SLICES_PER_GROUP - 1) {
float2 corr2 = make_float2(corr, corr);
float2 a[2];
#pragma unroll
for (int j = 0; j < 2; j++) {
float2 old = make_float2(acc32[si * VEC + 2 * j],
acc32[si * VEC + 2 * j + 1]);
a[j] = float2_mul(old, corr2);
}
float2 f_cache[AN][2];
#pragma unroll
for (int n = 0; n < AN; n++) {
tmem_load<4>(my_v_tmem + (si * N_CHUNK + n) * VEC,
f_cache[n]);
}
#pragma unroll
for (int n = 0; n < AN; n++) {
float2 wn = make_float2(w_n[n], w_n[n]);
#pragma unroll
for (int j = 0; j < 2; j++) {
a[j] = float2_fma(wn, f_cache[n][j], a[j]);
}
}
#pragma unroll
for (int j = 0; j < 2; j++) {
acc32[si * VEC + 2 * j] = a[j].x;
acc32[si * VEC + 2 * j + 1] = a[j].y;
}
continue;
}
}
int dt = si * CONSUMER_GROUPS + group;
if (dt >= NHT) continue;
float2 corr2 = make_float2(corr, corr);
float2 a[VEC / 2];
#pragma unroll
for (int j = 0; j < VEC / 2; j++) {
float2 old = make_float2(acc32[si * VEC + 2 * j],
acc32[si * VEC + 2 * j + 1]);
a[j] = float2_mul(old, corr2);
}
float2 f_cache[AN][VEC / 2];
#pragma unroll
for (int n = 0; n < AN; n++) {
tmem_load<VEC>(my_v_tmem + (si * N_CHUNK + n) * VEC, f_cache[n]);
}
#pragma unroll
for (int n = 0; n < AN; n++) {
float2 wn = make_float2(w_n[n], w_n[n]);
#pragma unroll
for (int j = 0; j < VEC / 2; j++) {
a[j] = float2_fma(wn, f_cache[n][j], a[j]);
}
}
#pragma unroll
for (int j = 0; j < VEC / 2; j++) {
acc32[si * VEC + 2 * j] = a[j].x;
acc32[si * VEC + 2 * j + 1] = a[j].y;
}
}
};
if constexpr (NC == 4) {
switch (an) {
case 4:
pass_B_body(std::integral_constant<int, 4>{});
break;
case 3:
pass_B_body(std::integral_constant<int, 3>{});
break;
case 2:
pass_B_body(std::integral_constant<int, 2>{});
break;
case 1:
pass_B_body(std::integral_constant<int, 1>{});
break;
default:
__builtin_unreachable();
}
} else if constexpr (NC == 3) {
switch (an) {
case 3:
pass_B_body(std::integral_constant<int, 3>{});
break;
case 2:
pass_B_body(std::integral_constant<int, 2>{});
break;
case 1:
pass_B_body(std::integral_constant<int, 1>{});
break;
default:
__builtin_unreachable();
}
} else {
static_assert(NC == 2);
switch (an) {
case 2:
pass_B_body(std::integral_constant<int, 2>{});
break;
case 1:
pass_B_body(std::integral_constant<int, 1>{});
break;
default:
__builtin_unreachable();
}
}
s_running = s_running * corr + w_sum;
m_running = m_new;
}
float inv_s = 1.f / s_running;
bf16_t* out_ptr = output + (long long)tb * H;
float2 output_sq_pair = {};
// When output RMSNorm is fused, the softmax denominator cancels:
// (acc / s) * rsqrt(mean((acc / s)^2) + eps)
// = acc * rsqrt(mean(acc^2) + eps * s^2).
#pragma unroll
for (int si = 0; si < SLICES_PER_GROUP; si++) {
if constexpr (H == 7168) {
if (si == SLICES_PER_GROUP - 1) {
int h_base = 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4;
uint2 packed;
auto* ov2 = reinterpret_cast<__nv_bfloat162*>(&packed);
float2 inv2 = make_float2(inv_s, inv_s);
#pragma unroll
for (int j = 0; j < 2; j++) {
float2 old = make_float2(acc32[si * VEC + 2 * j],
acc32[si * VEC + 2 * j + 1]);
if constexpr (HAS_OUTPUT_NORM) {
output_sq_pair = float2_fma(old, old, output_sq_pair);
} else {
float2 mixed = float2_mul(old, inv2);
ov2[j] = __float22bfloat162_rn(mixed);
}
}
if constexpr (!HAS_OUTPUT_NORM) {
*reinterpret_cast<uint2*>(out_ptr + h_base) = packed;
}
continue;
}
}
int dt = si * CONSUMER_GROUPS + group;
if (dt >= NHT) continue;
int h_base = dt * K_TILE + k_local;
uint4 packed;
auto* ov2 = reinterpret_cast<__nv_bfloat162*>(&packed);
float2 inv2 = make_float2(inv_s, inv_s);
#pragma unroll
for (int j = 0; j < VEC / 2; j++) {
float2 old =
make_float2(acc32[si * VEC + 2 * j], acc32[si * VEC + 2 * j + 1]);
if constexpr (HAS_OUTPUT_NORM) {
output_sq_pair = float2_fma(old, old, output_sq_pair);
} else {
float2 mixed = float2_mul(old, inv2);
ov2[j] = __float22bfloat162_rn(mixed);
}
}
if constexpr (!HAS_OUTPUT_NORM) {
*reinterpret_cast<uint4*>(out_ptr + h_base) = packed;
}
}
if constexpr (HAS_OUTPUT_NORM) {
if constexpr (OUTPUT_NORM_IN_SMEM) {
// The immutable weight copy is acquired once, at its first use.
if (tb == blockIdx.x) {
mbarrier_wait(plan.bar_output_norm_ready, 0);
}
}
float output_sq = output_sq_pair.x + output_sq_pair.y;
#pragma unroll
for (int offset = 16; offset > 0; offset >>= 1) {
output_sq += __shfl_xor_sync(0xffffffff, output_sq, offset);
}
if (lane == 0) {
plan.ws_stats[comp_wid][0] = make_float2(output_sq, 0.f);
}
named_barrier_sync(CONSUMER_THREADS, 0);
float total_sq = lane < CONSUMER_WARPS ? plan.ws_stats[lane][0].x : 0.f;
#pragma unroll
for (int offset = CONSUMER_WARPS / 2; offset > 0; offset >>= 1) {
total_sq +=
__shfl_down_sync(0xffffffff, total_sq, offset, CONSUMER_WARPS);
}
if (lane == 0) {
total_sq =
rsqrtf(total_sq / H + output_norm_eps * s_running * s_running);
}
float output_rsigma = __shfl_sync(0xffffffff, total_sq, 0);
#pragma unroll
for (int si = 0; si < SLICES_PER_GROUP; si++) {
if constexpr (H == 7168) {
if (si == SLICES_PER_GROUP - 1) {
int h_base = 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4;
uint2 packed;
auto* values = reinterpret_cast<bf16_t*>(&packed);
#pragma unroll
for (int j = 0; j < 4; j++) {
const bf16_t* weight_ptr =
OUTPUT_NORM_IN_SMEM ? output_norm_buf : output_norm_weight;
float weight = __bfloat162float(weight_ptr[h_base + j]);
values[j] = __float2bfloat16(acc32[si * VEC + j] *
output_rsigma * weight);
}
*reinterpret_cast<uint2*>(out_ptr + h_base) = packed;
continue;
}
}
int dt = si * CONSUMER_GROUPS + group;
if (dt >= NHT) continue;
int h_base = dt * K_TILE + k_local;
uint4 packed;
auto* values = reinterpret_cast<bf16_t*>(&packed);
#pragma unroll
for (int j = 0; j < VEC; j++) {
const bf16_t* weight_ptr =
OUTPUT_NORM_IN_SMEM ? output_norm_buf : output_norm_weight;
float weight = __bfloat162float(weight_ptr[h_base + j]);
values[j] =
__float2bfloat16(acc32[si * VEC + j] * output_rsigma * weight);
}
*reinterpret_cast<uint4*>(out_ptr + h_base) = packed;
}
}
}
}
cudaTriggerProgrammaticLaunchCompletion();
__syncthreads();
if (wid == 1) {
tmem_free(plan.tmem_base, TMEM_COLS_ALLOC);
}
#else
if (threadIdx.x == 0) {
printf("attn_res_fwd_online_v2_kernel requires sm_10x\n");
}
#endif
}
template <int H, int NC = N_CHUNK_DEFAULT, bool RELEASE_TMEM = false,
bool HAS_DELTA = false, bool HAS_OUTPUT_NORM = false,
bool OUTPUT_NORM_IN_SMEM = false>
static void launch_fwd(const bf16_t* block_residual, bf16_t* layer_residual,
const bf16_t* delta, const bf16_t* res_weight,
const bf16_t* rms_weight, bf16_t* output, int N, int T,
int B, float rms_eps, int num_sm, cudaStream_t stream,
const bf16_t* output_norm_weight = nullptr,
float output_norm_eps = 0.f, int block_stride_m = 0,
int block_stride_r = 0) {
constexpr size_t smem_size =
((size_t)CHUNK_DEPTH * (NC + (HAS_DELTA ? 1 : 0)) * H * sizeof(bf16_t) +
(OUTPUT_NORM_IN_SMEM ? (size_t)H * sizeof(bf16_t) : 0) +
sizeof(FwdSmemPlan<NC>) + 15) &
~size_t(15);
auto kernel =
&attn_res_fwd_online_v2_kernel<H, NC, RELEASE_TMEM, HAS_DELTA,
HAS_OUTPUT_NORM, OUTPUT_NORM_IN_SMEM>;
static bool attrs_set = false;
if (!attrs_set) {
if (smem_size > 48 * 1024) {
cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize,
smem_size);
}
attrs_set = true;
}
int grid = RELEASE_TMEM ? num_sm * 2 : num_sm;
cudaLaunchConfig_t config{};
config.gridDim = grid;
config.blockDim = BLK;
config.dynamicSmemBytes = smem_size;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.attrs = attrs;
config.numAttrs = 1;
cudaLaunchKernelEx(&config, kernel, block_residual, layer_residual, delta,
res_weight, rms_weight, output, N, T, B, block_stride_m,
block_stride_r, rms_eps, output_norm_weight,
output_norm_eps);
}
} // namespace fwd_prod_v2
} // namespace sm100
void kimi_k3_attn_res(torch::stable::Tensor& prefix,
torch::stable::Tensor const& delta,
torch::stable::Tensor const& blocks,
torch::stable::Tensor const& norm_weight,
torch::stable::Tensor const& qk_weight,
torch::stable::Tensor const& output_norm_weight,
torch::stable::Tensor& output, int64_t num_blocks,
double eps, double output_norm_eps) {
int const num_tokens = static_cast<int>(prefix.size(0));
int const device = prefix.get_device_index();
torch::stable::accelerator::DeviceGuard const device_guard(device);
cudaDeviceProp const* properties = get_device_prop();
STD_TORCH_CHECK(properties->major == 10,
"Kimi K3 AttnRes requires the SM100 family");
using namespace sm100::fwd_prod_v2;
// Two-source chunks and two resident CTAs are beneficial once setup is
// amortized by the long, full eight-block prefill workload.
if (num_blocks == 8 && num_tokens >= 4096) {
launch_fwd<7168, 2, true, true, true, true>(
static_cast<bf16_t const*>(blocks.data_ptr()),
static_cast<bf16_t*>(prefix.data_ptr()),
static_cast<bf16_t const*>(delta.data_ptr()),
static_cast<bf16_t const*>(qk_weight.data_ptr()),
static_cast<bf16_t const*>(norm_weight.data_ptr()),
static_cast<bf16_t*>(output.data_ptr()),
static_cast<int>(num_blocks) + 1, num_tokens, 1,
static_cast<float>(eps), properties->multiProcessorCount,
get_current_cuda_stream(device),
static_cast<bf16_t const*>(output_norm_weight.data_ptr()),
static_cast<float>(output_norm_eps), static_cast<int>(blocks.stride(0)),
static_cast<int>(blocks.stride(1)));
} else {
launch_fwd<7168, 4, false, true, true, true>(
static_cast<bf16_t const*>(blocks.data_ptr()),
static_cast<bf16_t*>(prefix.data_ptr()),
static_cast<bf16_t const*>(delta.data_ptr()),
static_cast<bf16_t const*>(qk_weight.data_ptr()),
static_cast<bf16_t const*>(norm_weight.data_ptr()),
static_cast<bf16_t*>(output.data_ptr()),
static_cast<int>(num_blocks) + 1, num_tokens, 1,
static_cast<float>(eps), properties->multiProcessorCount,
get_current_cuda_stream(device),
static_cast<bf16_t const*>(output_norm_weight.data_ptr()),
static_cast<float>(output_norm_eps), static_cast<int>(blocks.stride(0)),
static_cast<int>(blocks.stride(1)));
}
cudaError_t const error = cudaGetLastError();
STD_TORCH_CHECK(
error == cudaSuccess,
"Kimi K3 AttnRes kernel launch failed: ", cudaGetErrorString(error));
}
+11
View File
@@ -315,6 +315,17 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
std::optional<torch::stable::Tensor> index_q_out,
const std::string& kv_cache_dtype, bool skip_index_branch);
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
void kimi_k3_attn_res(torch::stable::Tensor& prefix,
torch::stable::Tensor const& delta,
torch::stable::Tensor const& blocks,
torch::stable::Tensor const& norm_weight,
torch::stable::Tensor const& qk_weight,
torch::stable::Tensor const& output_norm_weight,
torch::stable::Tensor& output, int64_t num_blocks,
double eps, double output_norm_eps);
#endif
// Sampler kernels (shared CUDA/ROCm)
void apply_repetition_penalties_(
torch::stable::Tensor& logits, const torch::stable::Tensor& prompt_mask,
+11
View File
@@ -468,6 +468,14 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"int block_size, Tensor!? q_out, Tensor!? index_q_out, "
"str kv_cache_dtype, bool skip_index_branch=False) -> ()");
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
ops.def(
"kimi_k3_attn_res("
"Tensor! prefix, Tensor delta, Tensor blocks, Tensor norm_weight, "
"Tensor qk_weight, Tensor output_norm_weight, Tensor! output, "
"int num_blocks, float eps, float output_norm_eps) -> ()");
#endif
// Apply repetition penalties to logits in-place.
ops.def(
"apply_repetition_penalties_(Tensor! logits, Tensor prompt_mask, "
@@ -693,6 +701,9 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
#endif
ops.impl("fused_minimax_m3_qknorm_rope_kv_insert",
TORCH_BOX(&fused_minimax_m3_qknorm_rope_kv_insert));
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res));
#endif
// Sampler kernels (shared CUDA/ROCm)
ops.impl("apply_repetition_penalties_",
+102
View File
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="AMD AttnRes requires ROCm",
)
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
storage = torch.randn(
*shape[:-1],
shape[-1] + padding,
device="cuda",
dtype=torch.bfloat16,
)
return storage[..., : shape[-1]]
def _reference(
prefix: torch.Tensor,
blocks: torch.Tensor,
norm_weight: torch.Tensor,
qk_weight: torch.Tensor,
num_blocks: int,
eps: float,
) -> torch.Tensor:
hidden_size = prefix.shape[-1]
values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1)
keys = F.rms_norm(values, (hidden_size,), norm_weight, eps)
probs = (keys @ qk_weight).softmax(dim=-1)
return torch.matmul(probs.unsqueeze(1), values).squeeze(1)
@pytest.mark.parametrize(
(
"num_tokens",
"num_blocks",
"block_capacity",
"hidden_size",
"row_padding",
),
[
pytest.param(0, 3, 5, 128, 0, id="empty"),
pytest.param(1, 1, 2, 128, 0, id="decode-single"),
pytest.param(17, 4, 6, 1024, 7, id="decode-padded"),
pytest.param(320, 8, 10, 7168, 0, id="prefill-full"),
],
)
def test_amd_attn_res_matches_reference(
num_tokens: int,
num_blocks: int,
block_capacity: int,
hidden_size: int,
row_padding: int,
) -> None:
eps = 1e-5
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=row_padding)
blocks = _randn_with_row_padding(
num_tokens,
block_capacity,
hidden_size,
padding=row_padding,
)
norm_weight = 1 + 0.1 * torch.randn(
hidden_size, device="cuda", dtype=torch.bfloat16
)
qk_weight = (
torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5
)
expected = _reference(
prefix,
blocks,
norm_weight,
qk_weight,
num_blocks,
eps,
)
original_prefix = prefix.clone()
original_blocks = blocks.clone()
actual = attn_res(
prefix,
blocks,
norm_weight,
qk_weight,
num_blocks,
eps,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
torch.testing.assert_close(prefix, original_prefix, atol=0, rtol=0)
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
assert actual.shape == prefix.shape
assert actual.is_contiguous()
+193
View File
@@ -0,0 +1,193 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.models.kimi_k3.nvidia.ops import attn_res
from vllm.platforms import current_platform
HIDDEN_SIZE = 7168
MAX_BLOCKS = 8
EPS = 1e-5
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
storage = torch.randn(
*shape[:-1],
shape[-1] + padding,
device="cuda",
dtype=torch.bfloat16,
)
return storage[..., : shape[-1]]
def _reference(
prefix: torch.Tensor,
delta: torch.Tensor | None,
blocks: torch.Tensor,
norm_weight: torch.Tensor,
qk_weight: torch.Tensor,
output_norm_weight: torch.Tensor | None,
num_blocks: int,
) -> tuple[torch.Tensor, torch.Tensor]:
if delta is not None:
prefix = prefix + delta
values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1)
keys = F.rms_norm(values, (HIDDEN_SIZE,), norm_weight, EPS)
probs = (keys @ qk_weight).softmax(dim=-1)
output = torch.matmul(probs.unsqueeze(1), values).squeeze(1)
if output_norm_weight is not None:
output = F.rms_norm(output, (HIDDEN_SIZE,), output_norm_weight, EPS)
return output, prefix
@pytest.mark.parametrize(
(
"num_tokens",
"num_blocks",
"row_padding",
"write_block",
"has_delta",
"backend",
),
[
pytest.param(1, 0, 0, True, False, "triton", id="triton-empty"),
pytest.param(1, 0, 0, True, True, "triton", id="triton-empty-add"),
pytest.param(17, 5, 7, True, False, "triton", id="triton-write"),
pytest.param(17, 5, 7, True, True, "triton", id="triton-write-add"),
pytest.param(3, 8, 0, False, False, "triton", id="triton-full"),
pytest.param(3, 8, 0, False, True, "triton", id="triton-full-add"),
pytest.param(320, 1, 0, False, True, "nvidia", id="nvidia-1"),
pytest.param(320, 4, 0, False, True, "nvidia", id="nvidia-4"),
pytest.param(320, 8, 0, False, True, "nvidia", id="nvidia-8"),
],
)
def test_attn_res(
num_tokens: int,
num_blocks: int,
row_padding: int,
write_block: bool,
has_delta: bool,
backend: str,
):
if backend == "nvidia" and not current_platform.is_device_capability_family(100):
pytest.skip("NVIDIA AttnRes requires the SM100 family")
prefix = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
delta = (
_randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
if has_delta
else None
)
blocks = _randn_with_row_padding(
num_tokens, MAX_BLOCKS, HIDDEN_SIZE, padding=row_padding
)
norm_weight = 1 + 0.1 * torch.randn(
HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
)
qk_weight = (
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
)
output_norm_weight = 1 + 0.1 * torch.randn(
HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
)
original_blocks = blocks.clone()
expected, expected_prefix = _reference(
prefix.clone(),
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
)
block_write_idx = num_blocks if write_block else -1
actual = attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
block_write_idx,
EPS,
EPS,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0)
if write_block:
original_blocks[:, block_write_idx].copy_(expected_prefix)
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
assert actual.is_contiguous()
@pytest.mark.parametrize("num_blocks", range(MAX_BLOCKS + 1))
def test_attn_res_block_counts(num_blocks: int):
prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
blocks = torch.randn(
1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
)
norm_weight = torch.ones(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
qk_weight = (
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
)
output_norm_weight = torch.ones_like(norm_weight)
expected, _ = _reference(
prefix.clone(),
None,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
)
actual = attn_res(
prefix,
None,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
-1,
EPS,
EPS,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
def test_attn_res_without_output_norm():
prefix = torch.randn(7, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
delta = torch.randn_like(prefix)
blocks = torch.randn(
7, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
)
norm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
qk_weight = (
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
)
expected, _ = _reference(
prefix.clone(), delta, blocks, norm_weight, qk_weight, None, MAX_BLOCKS
)
actual = attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
None,
MAX_BLOCKS,
-1,
EPS,
0.0,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
+27
View File
@@ -2746,6 +2746,33 @@ def concat_and_cache_mla(
)
def kimi_k3_attn_res(
prefix: torch.Tensor,
delta: torch.Tensor,
blocks: torch.Tensor,
norm_weight: torch.Tensor,
qk_weight: torch.Tensor,
output_norm_weight: torch.Tensor,
num_blocks: int,
eps: float,
output_norm_eps: float,
) -> torch.Tensor:
output = torch.empty_like(prefix)
torch.ops._C.kimi_k3_attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
output,
num_blocks,
eps,
output_norm_eps,
)
return output
def concat_and_cache_mla_rope_fused(
positions: torch.Tensor,
q_pe: torch.Tensor,
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+132
View File
@@ -0,0 +1,132 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
#
# This file contains code adapted from the flash-linear-attention project.
# The original source code was licensed under the MIT license and included
# the following copyright notice:
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
import torch
from vllm.triton_utils import tl, triton
@triton.jit
def _attn_res_kernel(
prefix_ptr,
blocks_ptr,
norm_weight_ptr,
qk_weight_ptr,
output_ptr,
stride_prefix_m: tl.constexpr,
stride_block_m: tl.constexpr,
stride_block_r: tl.constexpr,
stride_output_m: tl.constexpr,
num_blocks: tl.constexpr,
hidden_size: tl.constexpr,
eps: tl.constexpr,
BLOCK_L: tl.constexpr,
BLOCK_D: tl.constexpr,
):
row_idx = tl.program_id(0).to(tl.int64)
d_offsets = tl.max_contiguous(tl.arange(0, BLOCK_D), BLOCK_D)
d_mask = d_offsets < hidden_size
prefix = tl.load(
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
mask=d_mask,
other=0.0,
).to(tl.float32)
input_qk_weight = tl.load(norm_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(
tl.float32
) * tl.load(qk_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(tl.float32)
max_logit = tl.full((), -float("inf"), tl.float32)
denominator = tl.zeros((), tl.float32)
mixed = tl.zeros((BLOCK_D,), tl.float32)
num_sources = num_blocks + 1
for source_tile in range(tl.cdiv(num_sources, BLOCK_L)):
source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L)
source_mask = source_offsets < num_sources
is_prefix = source_offsets == num_blocks
block_ptrs = (
blocks_ptr
+ row_idx * stride_block_m
+ source_offsets[:, None] * stride_block_r
+ d_offsets[None, :]
)
block_values = tl.load(
block_ptrs,
mask=(source_mask[:, None] & ~is_prefix[:, None] & d_mask[None, :]),
other=0.0,
eviction_policy="evict_first",
).to(tl.float32)
values = tl.where(is_prefix[:, None], prefix[None, :], block_values)
reciprocal_std = tl.rsqrt(
tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps
)
logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std
scores = tl.where(source_mask, logits, -float("inf"))
new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0))
old_scale = tl.exp(max_logit - new_max_logit)
block_scales = tl.exp(scores - new_max_logit)
denominator = denominator * old_scale + tl.sum(block_scales, axis=0)
mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0)
max_logit = new_max_logit
output = mixed / denominator
tl.store(
output_ptr + row_idx * stride_output_m + d_offsets,
output,
mask=d_mask,
)
def attn_res(
prefix: torch.Tensor,
blocks: torch.Tensor,
norm_weight: torch.Tensor,
qk_weight: torch.Tensor,
num_blocks: int,
eps: float,
) -> torch.Tensor:
num_tokens, hidden_size = prefix.shape
assert 0 < num_blocks <= blocks.shape[1]
assert blocks.shape[0] == num_tokens
assert norm_weight.numel() == hidden_size
assert qk_weight.numel() == hidden_size
assert prefix.stride(-1) == 1
assert blocks.stride(-1) == 1
assert norm_weight.stride(-1) == 1
assert qk_weight.stride(-1) == 1
output = prefix.new_empty(prefix.shape)
if num_tokens == 0:
return output
if num_tokens >= 256 or num_blocks <= 1:
block_l, num_warps = 1, 4
else:
block_l, num_warps = 4, 8
_attn_res_kernel[(num_tokens,)](
prefix,
blocks,
norm_weight,
qk_weight,
output,
prefix.stride(0),
blocks.stride(0),
blocks.stride(1),
output.stride(0),
num_blocks,
hidden_size,
eps,
BLOCK_L=block_l,
BLOCK_D=triton.next_power_of_2(hidden_size),
num_warps=num_warps,
num_stages=2,
)
return output
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .attn_res import attn_res
__all__ = ["attn_res"]
+245
View File
@@ -0,0 +1,245 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
#
# This file contains code adapted from the flash-linear-attention project.
# The original source code was licensed under the MIT license and included
# the following copyright notice:
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
import torch
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
def get_attn_res_triton_warmup_profiles(
max_blocks: int,
) -> tuple[tuple[int, bool, int, bool], ...]:
"""Return the small-batch profiles that bypass the native kernel."""
profiles = [
(num_blocks, False, -1, True) for num_blocks in range(2, max_blocks + 1)
]
profiles.extend(
(block_write_idx, True, block_write_idx, True)
for block_write_idx in range(2, max_blocks)
)
profiles.append((max_blocks, True, -1, False))
return tuple(profiles)
@triton.jit
def _attn_res_kernel(
prefix_ptr,
delta_ptr,
blocks_ptr,
norm_weight_ptr,
qk_weight_ptr,
output_norm_weight_ptr,
output_ptr,
stride_prefix_m: tl.constexpr,
stride_delta_m: tl.constexpr,
stride_block_m: tl.constexpr,
stride_block_r: tl.constexpr,
stride_output_m: tl.constexpr,
num_blocks: tl.constexpr,
hidden_size: tl.constexpr,
block_write_idx: tl.constexpr,
eps: tl.constexpr,
output_norm_eps: tl.constexpr,
HAS_DELTA: tl.constexpr,
WRITE_BLOCK: tl.constexpr,
APPLY_OUTPUT_NORM: tl.constexpr,
BLOCK_L: tl.constexpr,
BLOCK_D: tl.constexpr,
launch_pdl: tl.constexpr,
):
row_idx = tl.program_id(0).to(tl.int64)
d_offsets = tl.max_contiguous(tl.arange(0, BLOCK_D), BLOCK_D)
d_mask = d_offsets < hidden_size
if launch_pdl:
tl.extra.cuda.gdc_wait()
updated_prefix = tl.load(
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
mask=d_mask,
other=0.0,
).to(tl.float32)
if HAS_DELTA:
delta = tl.load(
delta_ptr + row_idx * stride_delta_m + d_offsets,
mask=d_mask,
other=0.0,
).to(tl.float32)
updated_prefix += delta
# Match the BF16 prefix-add result before using it as a residual source.
updated_prefix = updated_prefix.to(prefix_ptr.dtype.element_ty).to(tl.float32)
tl.store(
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
updated_prefix,
mask=d_mask,
)
if WRITE_BLOCK:
tl.store(
blocks_ptr
+ row_idx * stride_block_m
+ block_write_idx * stride_block_r
+ d_offsets,
updated_prefix,
mask=d_mask,
)
# With only the prefix source, the AttnRes softmax is exactly one.
if num_blocks == 0:
mixed = updated_prefix
else:
# Reloading avoids keeping the full prefix vector live across the loop.
if HAS_DELTA:
tl.debug_barrier()
input_qk_weight = tl.load(
norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
).to(tl.float32) * tl.load(
qk_weight_ptr + d_offsets, mask=d_mask, other=0.0
).to(tl.float32)
max_logit = tl.full((), -float("inf"), tl.float32)
denominator = tl.zeros((), tl.float32)
mixed = tl.zeros((BLOCK_D,), tl.float32)
num_sources = num_blocks + 1
for source_tile in range(tl.cdiv(num_sources, BLOCK_L)):
source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L)
source_mask = source_offsets < num_sources
is_prefix = source_offsets == num_blocks
block_ptrs = (
blocks_ptr
+ row_idx * stride_block_m
+ source_offsets[:, None] * stride_block_r
+ d_offsets[None, :]
)
prefix_ptrs = (
prefix_ptr
+ row_idx * stride_prefix_m
+ source_offsets[:, None] * 0
+ d_offsets[None, :]
)
value_ptrs = tl.where(is_prefix[:, None], prefix_ptrs, block_ptrs)
values = tl.load(
value_ptrs,
mask=source_mask[:, None] & d_mask[None, :],
other=0.0,
eviction_policy="evict_first",
).to(tl.float32)
reciprocal_std = tl.rsqrt(
tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps
)
logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std
scores = tl.where(source_mask, logits, -float("inf"))
new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0))
old_scale = tl.exp(max_logit - new_max_logit)
block_scales = tl.exp(scores - new_max_logit)
denominator = denominator * old_scale + tl.sum(block_scales, axis=0)
mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0)
max_logit = new_max_logit
mixed /= denominator
output = mixed
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
if APPLY_OUTPUT_NORM:
output_reciprocal_std = tl.rsqrt(
tl.sum(tl.where(d_mask, mixed * mixed, 0.0), axis=0) * (1.0 / hidden_size)
+ output_norm_eps
)
output_norm_weight = tl.load(
output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
).to(tl.float32)
output = mixed * output_reciprocal_std * output_norm_weight
tl.store(
output_ptr + row_idx * stride_output_m + d_offsets,
output,
mask=d_mask,
)
def attn_res(
prefix: torch.Tensor,
delta: torch.Tensor | None,
blocks: torch.Tensor,
norm_weight: torch.Tensor,
qk_weight: torch.Tensor,
output_norm_weight: torch.Tensor | None,
num_blocks: int,
block_write_idx: int,
eps: float,
output_norm_eps: float,
) -> torch.Tensor:
num_tokens, hidden_size = prefix.shape
assert prefix.stride(-1) == 1
assert delta is None or delta.stride(-1) == 1
assert blocks.stride(-1) == 1
assert norm_weight.stride(-1) == 1
assert qk_weight.stride(-1) == 1
assert output_norm_weight is None or output_norm_weight.stride(-1) == 1
# The in-tree NVIDIA kernel covers the common fused-add + output-norm path;
# Triton handles block boundaries and final pre-norm output.
if (
hidden_size == 7168
and delta is not None
and output_norm_weight is not None
and num_blocks > 0
and block_write_idx < 0
and current_platform.is_device_capability_family(100)
):
return ops.kimi_k3_attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
eps,
output_norm_eps,
)
output = prefix.new_empty(prefix.shape)
# Tuned on GB300: source tiling helps decode, while one-source tiles scale
# better for prefill.
# Keep get_attn_res_triton_warmup_profiles in sync with these fallbacks.
if num_tokens >= 256 or num_blocks <= 1:
block_l, num_warps = 1, 4
else:
block_l, num_warps = 4, 8
_attn_res_kernel[(num_tokens,)](
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
output,
prefix.stride(0),
0 if delta is None else delta.stride(0),
blocks.stride(0),
blocks.stride(1),
output.stride(0),
num_blocks,
hidden_size,
block_write_idx,
eps,
output_norm_eps,
HAS_DELTA=delta is not None,
WRITE_BLOCK=block_write_idx >= 0,
APPLY_OUTPUT_NORM=output_norm_weight is not None,
BLOCK_L=block_l,
BLOCK_D=triton.next_power_of_2(hidden_size),
num_warps=num_warps,
num_stages=2,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return output