[Kimi-K3] Add AttnRes kernels (#50090)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -61,6 +61,18 @@ steps:
|
||||
# FA4 kernel tests require SM100; the suite skips them elsewhere.
|
||||
- pytest -v -s models/inkling
|
||||
|
||||
- label: Kimi K3 Unit Tests (B200)
|
||||
key: kimi-k3-unit-tests-b200
|
||||
timeout_in_minutes: 40
|
||||
device: b200-k8s
|
||||
source_file_dependencies:
|
||||
- vllm/models/kimi_k3/
|
||||
- csrc/libtorch_stable/kimi_k3/
|
||||
- tests/models/kimi_k3/
|
||||
commands:
|
||||
# The native NVIDIA AttnRes kernel requires the SM100 family.
|
||||
- pytest -v -s models/kimi_k3
|
||||
|
||||
- label: Basic Models Test (Other CPU) # 5min
|
||||
key: basic-models-test-other-cpu
|
||||
depends_on:
|
||||
|
||||
@@ -1079,6 +1079,24 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
set(MLA_ARCHS)
|
||||
endif()
|
||||
|
||||
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
|
||||
cuda_archs_loose_intersection(KIMI_K3_ATTN_RES_ARCHS
|
||||
"10.0f" "${CUDA_ARCHS}")
|
||||
endif()
|
||||
if(KIMI_K3_ATTN_RES_ARCHS)
|
||||
set(KIMI_K3_ATTN_RES_SRC
|
||||
"csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu")
|
||||
set_gencode_flags_for_srcs(
|
||||
SRCS "${KIMI_K3_ATTN_RES_SRC}"
|
||||
CUDA_ARCHS "${KIMI_K3_ATTN_RES_ARCHS}")
|
||||
set_property(SOURCE ${KIMI_K3_ATTN_RES_SRC} APPEND PROPERTY
|
||||
COMPILE_OPTIONS
|
||||
"$<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr;--expt-extended-lambda;--use_fast_math>")
|
||||
list(APPEND VLLM_STABLE_EXT_SRC "${KIMI_K3_ATTN_RES_SRC}")
|
||||
message(STATUS
|
||||
"Building Kimi K3 AttnRes for archs: ${KIMI_K3_ATTN_RES_ARCHS}")
|
||||
endif()
|
||||
|
||||
# Hadacore kernels
|
||||
cuda_archs_loose_intersection(HADACORE_ARCHS "8.0+PTX;9.0+PTX" "${CUDA_ARCHS}")
|
||||
if(HADACORE_ARCHS)
|
||||
@@ -1120,6 +1138,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
target_compile_definitions(_C_stable_libtorch PRIVATE
|
||||
VLLM_ENABLE_COOPERATIVE_TOPK=1)
|
||||
endif()
|
||||
if(KIMI_K3_ATTN_RES_ARCHS)
|
||||
target_compile_definitions(_C_stable_libtorch PRIVATE
|
||||
VLLM_ENABLE_KIMI_K3_ATTN_RES=1)
|
||||
endif()
|
||||
# Needed by CUTLASS kernels
|
||||
target_compile_definitions(_C_stable_libtorch PRIVATE
|
||||
CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
|
||||
|
||||
@@ -0,0 +1,954 @@
|
||||
/*
|
||||
* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
||||
*/
|
||||
|
||||
// Production AttnRes forward for Blackwell (SM100).
|
||||
//
|
||||
// Warp-specialized online softmax + residual + RMSNorm:
|
||||
// - 1 producer warp issues cp.async.bulk row loads into shared memory.
|
||||
// - 8 consumer warps compute reductions and output.
|
||||
// - Q=res_weight*rms_weight remains in registers across persistent tokens.
|
||||
// - V rows are converted once and cached as FP32 in TMEM between passes.
|
||||
//
|
||||
// Integration contract: Kimi K3 H=7168, 1<=num_blocks<=8, and token-major
|
||||
// block residual storage.
|
||||
|
||||
#include "../torch_utils.h"
|
||||
|
||||
#include <cfloat>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cuda_runtime.h>
|
||||
#include <type_traits>
|
||||
|
||||
using bf16_t = __nv_bfloat16;
|
||||
|
||||
namespace sm100 {
|
||||
namespace fwd_prod_v2 {
|
||||
|
||||
constexpr int K_TILE = 1024;
|
||||
constexpr int N_CHUNK_DEFAULT = 4;
|
||||
constexpr int CHUNK_DEPTH = 2;
|
||||
constexpr int BLK = 288; // 1 producer warp + 8 consumer warps
|
||||
constexpr int CONSUMER_THREADS = BLK - 32; // 256
|
||||
constexpr int CONSUMER_WARPS = CONSUMER_THREADS / 32;
|
||||
constexpr int CONSUMER_GROUPS = 2; // two 128-thread consumer groups
|
||||
constexpr int CONSUMER_THREADS_PER_GROUP = CONSUMER_THREADS / CONSUMER_GROUPS;
|
||||
constexpr int FIRST_USER_NAMED_BARRIER = 8;
|
||||
|
||||
__device__ __forceinline__ const bf16_t* residual_addr(
|
||||
const bf16_t* block_res, const bf16_t* layer_res, int source, int N,
|
||||
int token, int block_stride_m, int block_stride_r, int H) {
|
||||
if (source < N - 1) {
|
||||
return block_res + static_cast<long long>(token) * block_stride_m +
|
||||
source * block_stride_r;
|
||||
}
|
||||
return layer_res + static_cast<long long>(token) * H;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t elect_one_sync() {
|
||||
uint32_t pred = 0;
|
||||
uint32_t laneid = 0;
|
||||
asm volatile(
|
||||
"{\n"
|
||||
".reg .b32 %%rx;\n"
|
||||
".reg .pred %%px;\n"
|
||||
" elect.sync %%rx|%%px, %2;\n"
|
||||
"@%%px mov.s32 %1, 1;\n"
|
||||
" mov.s32 %0, %%rx;\n"
|
||||
"}\n"
|
||||
: "+r"(laneid), "+r"(pred)
|
||||
: "r"(0xffffffff));
|
||||
return pred;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void mbarrier_init(uint64_t& barrier,
|
||||
int thread_count) {
|
||||
uint32_t const barrier_addr =
|
||||
static_cast<uint32_t>(__cvta_generic_to_shared(&barrier));
|
||||
asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" ::"r"(barrier_addr),
|
||||
"r"(thread_count));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void mbarrier_expect_tx(uint64_t& barrier,
|
||||
uint32_t bytes) {
|
||||
uint32_t const barrier_addr =
|
||||
static_cast<uint32_t>(__cvta_generic_to_shared(&barrier));
|
||||
asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" ::"r"(
|
||||
barrier_addr),
|
||||
"r"(bytes));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void mbarrier_wait(uint64_t& barrier, int phase) {
|
||||
uint32_t const barrier_addr =
|
||||
static_cast<uint32_t>(__cvta_generic_to_shared(&barrier));
|
||||
asm volatile(
|
||||
"{\n"
|
||||
".reg .pred p;\n"
|
||||
"WAIT:\n"
|
||||
"mbarrier.try_wait.parity.shared::cta.b64 p, [%0], %1;\n"
|
||||
"@p bra DONE;\n"
|
||||
"bra WAIT;\n"
|
||||
"DONE:\n"
|
||||
"}\n" ::"r"(barrier_addr),
|
||||
"r"(phase));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void mbarrier_arrive(uint64_t& barrier) {
|
||||
uint32_t const barrier_addr =
|
||||
static_cast<uint32_t>(__cvta_generic_to_shared(&barrier));
|
||||
asm volatile(
|
||||
"{\n"
|
||||
".reg .b64 state;\n"
|
||||
"mbarrier.arrive.shared::cta.b64 state, [%0];\n"
|
||||
"}\n" ::"r"(barrier_addr));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void fence_mbarrier_init() {
|
||||
asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void named_barrier_sync(uint32_t num_threads,
|
||||
uint32_t user_barrier_id) {
|
||||
asm volatile(
|
||||
"bar.sync %0, %1;" ::"r"(user_barrier_id + FIRST_USER_NAMED_BARRIER),
|
||||
"r"(num_threads)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tmem_allocate(int num_columns, uint32_t* dst) {
|
||||
uint32_t const dst_addr =
|
||||
static_cast<uint32_t>(__cvta_generic_to_shared(dst));
|
||||
asm volatile(
|
||||
"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"(
|
||||
dst_addr),
|
||||
"r"(num_columns));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tmem_free(uint32_t tmem_ptr, int num_columns) {
|
||||
asm volatile(
|
||||
"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" ::"r"(tmem_ptr),
|
||||
"r"(num_columns));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tmem_release_allocation_lock() {
|
||||
asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;");
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tmem_store_wait() {
|
||||
asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory");
|
||||
}
|
||||
|
||||
template <int N, typename T>
|
||||
__device__ __forceinline__ void tmem_load(uint32_t src_addr, T* dst) {
|
||||
uint32_t* values = reinterpret_cast<uint32_t*>(dst);
|
||||
if constexpr (N == 8) {
|
||||
asm volatile(
|
||||
"tcgen05.ld.sync.aligned.32x32b.x8.b32"
|
||||
"{%0, %1, %2, %3, %4, %5, %6, %7}, [%8];\n"
|
||||
: "=r"(values[0]), "=r"(values[1]), "=r"(values[2]), "=r"(values[3]),
|
||||
"=r"(values[4]), "=r"(values[5]), "=r"(values[6]), "=r"(values[7])
|
||||
: "r"(src_addr));
|
||||
} else {
|
||||
static_assert(N == 4, "AttnRes TMEM helpers support x4 and x8");
|
||||
asm volatile(
|
||||
"tcgen05.ld.sync.aligned.32x32b.x4.b32"
|
||||
"{%0, %1, %2, %3}, [%4];\n"
|
||||
: "=r"(values[0]), "=r"(values[1]), "=r"(values[2]), "=r"(values[3])
|
||||
: "r"(src_addr));
|
||||
}
|
||||
}
|
||||
|
||||
template <int N, typename T>
|
||||
__device__ __forceinline__ void tmem_store(uint32_t dst_addr, T* src) {
|
||||
uint32_t* values = reinterpret_cast<uint32_t*>(src);
|
||||
if constexpr (N == 8) {
|
||||
asm volatile(
|
||||
"tcgen05.st.sync.aligned.32x32b.x8.b32"
|
||||
"[%8], {%0, %1, %2, %3, %4, %5, %6, %7};\n" ::"r"(values[0]),
|
||||
"r"(values[1]), "r"(values[2]), "r"(values[3]), "r"(values[4]),
|
||||
"r"(values[5]), "r"(values[6]), "r"(values[7]), "r"(dst_addr));
|
||||
} else {
|
||||
static_assert(N == 4, "AttnRes TMEM helpers support x4 and x8");
|
||||
asm volatile(
|
||||
"tcgen05.st.sync.aligned.32x32b.x4.b32"
|
||||
"[%4], {%0, %1, %2, %3};\n" ::"r"(values[0]),
|
||||
"r"(values[1]), "r"(values[2]), "r"(values[3]), "r"(dst_addr));
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float2 float2_add(const float2& a, const float2& b) {
|
||||
float2 result;
|
||||
asm volatile("add.rn.f32x2 %0, %1, %2;\n"
|
||||
: "=l"(reinterpret_cast<uint64_t&>(result))
|
||||
: "l"(reinterpret_cast<uint64_t const&>(a)),
|
||||
"l"(reinterpret_cast<uint64_t const&>(b)));
|
||||
return result;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float2 float2_mul(const float2& a, const float2& b) {
|
||||
float2 result;
|
||||
asm volatile("mul.f32x2 %0, %1, %2;\n"
|
||||
: "=l"(reinterpret_cast<uint64_t&>(result))
|
||||
: "l"(reinterpret_cast<uint64_t const&>(a)),
|
||||
"l"(reinterpret_cast<uint64_t const&>(b)));
|
||||
return result;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float2 float2_fma(const float2& a, const float2& b,
|
||||
const float2& c) {
|
||||
float2 result;
|
||||
asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n"
|
||||
: "=l"(reinterpret_cast<uint64_t&>(result))
|
||||
: "l"(reinterpret_cast<uint64_t const&>(a)),
|
||||
"l"(reinterpret_cast<uint64_t const&>(b)),
|
||||
"l"(reinterpret_cast<uint64_t const&>(c)));
|
||||
return result;
|
||||
}
|
||||
|
||||
template <int NC>
|
||||
struct FwdSmemPlan {
|
||||
alignas(16) uint64_t bar_ready[CHUNK_DEPTH];
|
||||
alignas(16) uint64_t bar_consumed[CHUNK_DEPTH];
|
||||
alignas(16) uint64_t bar_output_norm_ready;
|
||||
alignas(16) float2 ws_stats[CONSUMER_WARPS][NC];
|
||||
uint32_t tmem_base;
|
||||
};
|
||||
|
||||
__device__ __forceinline__ void cp_async_bulk(void* smem_dst,
|
||||
const void* gmem_src, int bytes,
|
||||
uint64_t& mbar) {
|
||||
uint32_t const s = static_cast<uint32_t>(__cvta_generic_to_shared(smem_dst));
|
||||
uint32_t const m = static_cast<uint32_t>(__cvta_generic_to_shared(&mbar));
|
||||
asm volatile(
|
||||
"cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [%0], "
|
||||
"[%1], %2, [%3];\n" ::"r"(s),
|
||||
"l"(gmem_src), "r"(bytes), "r"(m)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
template <int H, int NC = N_CHUNK_DEFAULT, bool RELEASE_TMEM = false,
|
||||
bool HAS_DELTA = false, bool HAS_OUTPUT_NORM = false,
|
||||
bool OUTPUT_NORM_IN_SMEM = false>
|
||||
__global__ void __launch_bounds__(BLK, 1) attn_res_fwd_online_v2_kernel(
|
||||
const bf16_t* __restrict__ block_res, bf16_t* __restrict__ layer_res,
|
||||
const bf16_t* __restrict__ delta, const bf16_t* __restrict__ res_w,
|
||||
const bf16_t* __restrict__ rms_w, bf16_t* __restrict__ output, int N, int T,
|
||||
int B, int block_stride_m, int block_stride_r, float rms_eps,
|
||||
const bf16_t* __restrict__ output_norm_weight, float output_norm_eps) {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && __CUDA_ARCH__ < 1100
|
||||
constexpr float LOG2_E = 1.4426950408889634f;
|
||||
constexpr int N_CHUNK = NC;
|
||||
// The two-source specialization only consumes half of the TMEM columns.
|
||||
constexpr int TMEM_COLS_ALLOC = NC == 2 ? 128 : 256;
|
||||
constexpr int NUM_BUFS = CHUNK_DEPTH * NC;
|
||||
constexpr int NHT = H / K_TILE;
|
||||
constexpr int SLICES_PER_GROUP =
|
||||
(NHT + CONSUMER_GROUPS - 1) / CONSUMER_GROUPS;
|
||||
constexpr int VEC = 8;
|
||||
constexpr int ACC_PER_THREAD = H == 7168 ? 28 : SLICES_PER_GROUP * VEC;
|
||||
constexpr int TMEM_V_COLS_PER_GROUP = SLICES_PER_GROUP * N_CHUNK * VEC;
|
||||
constexpr int TMEM_V_COLS_TOTAL = CONSUMER_GROUPS * TMEM_V_COLS_PER_GROUP;
|
||||
static_assert(TMEM_V_COLS_TOTAL <= TMEM_COLS_ALLOC);
|
||||
static_assert(H >= 4096 && H <= 8192);
|
||||
static_assert(H % K_TILE == 0);
|
||||
|
||||
const int tid = threadIdx.x;
|
||||
const int wid = tid >> 5;
|
||||
const int lane = tid & 31;
|
||||
const int TB = T * B;
|
||||
const int num_ctas = gridDim.x;
|
||||
const int num_chunks = (N + N_CHUNK - 1) / N_CHUNK;
|
||||
|
||||
const int comp_wid = wid - 1;
|
||||
const int comp_tid = tid - 32;
|
||||
const int group = (comp_wid >= 4) ? 1 : 0;
|
||||
const int ct_in_group =
|
||||
(comp_tid >= 0) ? (comp_tid & (CONSUMER_THREADS_PER_GROUP - 1)) : -1;
|
||||
const int k_local = ct_in_group * VEC;
|
||||
|
||||
constexpr size_t V_BYTES = (size_t)NUM_BUFS * H * sizeof(bf16_t);
|
||||
constexpr size_t DELTA_BYTES =
|
||||
HAS_DELTA ? (size_t)CHUNK_DEPTH * H * sizeof(bf16_t) : 0;
|
||||
constexpr size_t OUTPUT_NORM_BYTES =
|
||||
OUTPUT_NORM_IN_SMEM ? (size_t)H * sizeof(bf16_t) : 0;
|
||||
extern __shared__ __align__(16) char smem_raw[];
|
||||
bf16_t* v_bufs = reinterpret_cast<bf16_t*>(smem_raw); // [NUM_BUFS][H]
|
||||
bf16_t* delta_bufs = reinterpret_cast<bf16_t*>(smem_raw + V_BYTES);
|
||||
bf16_t* output_norm_buf =
|
||||
reinterpret_cast<bf16_t*>(smem_raw + V_BYTES + DELTA_BYTES);
|
||||
FwdSmemPlan<NC>& plan = *reinterpret_cast<FwdSmemPlan<NC>*>(
|
||||
smem_raw + V_BYTES + DELTA_BYTES + OUTPUT_NORM_BYTES);
|
||||
|
||||
auto slot_of = [](long long gci, int n) {
|
||||
return (int)(gci % CHUNK_DEPTH) * N_CHUNK + n;
|
||||
};
|
||||
auto phase_of = [](long long gci) { return (int)((gci / CHUNK_DEPTH) & 1); };
|
||||
auto buf_ptr = [&](int slot) -> bf16_t* { return v_bufs + slot * H; };
|
||||
auto delta_buf_ptr = [&](int chunk_slot) -> bf16_t* {
|
||||
return delta_bufs + chunk_slot * H;
|
||||
};
|
||||
|
||||
if (wid == 0 && elect_one_sync()) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < CHUNK_DEPTH; i++) {
|
||||
mbarrier_init(plan.bar_ready[i], 1);
|
||||
mbarrier_init(plan.bar_consumed[i], CONSUMER_WARPS);
|
||||
}
|
||||
if constexpr (OUTPUT_NORM_IN_SMEM) {
|
||||
mbarrier_init(plan.bar_output_norm_ready, 1);
|
||||
}
|
||||
fence_mbarrier_init();
|
||||
}
|
||||
|
||||
// gdc wait BEFORE tmem alloc
|
||||
cudaGridDependencySynchronize();
|
||||
|
||||
if (wid == 1) {
|
||||
tmem_allocate(TMEM_COLS_ALLOC, &plan.tmem_base);
|
||||
if constexpr (RELEASE_TMEM) {
|
||||
tmem_release_allocation_lock();
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
if constexpr (OUTPUT_NORM_IN_SMEM) {
|
||||
if (wid == 0 && elect_one_sync()) {
|
||||
mbarrier_expect_tx(plan.bar_output_norm_ready, H * (int)sizeof(bf16_t));
|
||||
cp_async_bulk(output_norm_buf, output_norm_weight, H * sizeof(bf16_t),
|
||||
plan.bar_output_norm_ready);
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t my_v_tmem =
|
||||
comp_tid >= 0 ? plan.tmem_base + group * TMEM_V_COLS_PER_GROUP : 0;
|
||||
float q_cache[ACC_PER_THREAD];
|
||||
if (comp_tid >= 0) {
|
||||
#pragma unroll
|
||||
for (int si = 0; si < SLICES_PER_GROUP; si++) {
|
||||
if constexpr (H == 7168) {
|
||||
if (si == SLICES_PER_GROUP - 1) {
|
||||
int h_base = 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 4; j++) {
|
||||
int h = h_base + j;
|
||||
q_cache[si * VEC + j] =
|
||||
__bfloat162float(rms_w[h]) * __bfloat162float(res_w[h]);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
int dt = si * CONSUMER_GROUPS + group;
|
||||
if (dt >= NHT) continue;
|
||||
int h_base = dt * K_TILE + k_local;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC; j++) {
|
||||
int h = h_base + j;
|
||||
q_cache[si * VEC + j] =
|
||||
__bfloat162float(rms_w[h]) * __bfloat162float(res_w[h]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (wid == 0) {
|
||||
if (elect_one_sync()) {
|
||||
long long gci = 0;
|
||||
for (int tb = blockIdx.x; tb < TB; tb += num_ctas) {
|
||||
const int t = tb / B;
|
||||
for (int ci = 0; ci < num_chunks; ci++, gci++) {
|
||||
int ns = ci * N_CHUNK;
|
||||
int an = min(N_CHUNK, N - ns);
|
||||
int chunk_slot = (int)(gci % CHUNK_DEPTH);
|
||||
int pc = phase_of(gci);
|
||||
mbarrier_wait(plan.bar_consumed[chunk_slot], pc ^ 1);
|
||||
int transaction_bytes = an * H * (int)sizeof(bf16_t);
|
||||
if constexpr (HAS_DELTA) {
|
||||
int prefix_n = N - 1 - ns;
|
||||
if (prefix_n >= 0 && prefix_n < an) {
|
||||
transaction_bytes += H * (int)sizeof(bf16_t);
|
||||
}
|
||||
}
|
||||
mbarrier_expect_tx(plan.bar_ready[chunk_slot], transaction_bytes);
|
||||
#pragma unroll
|
||||
for (int n = 0; n < N_CHUNK; n++) {
|
||||
if (n >= an) continue;
|
||||
int slot = slot_of(gci, n);
|
||||
const bf16_t* src =
|
||||
residual_addr(block_res, layer_res, ns + n, N, t,
|
||||
block_stride_m, block_stride_r, H);
|
||||
cp_async_bulk(buf_ptr(slot), src, H * sizeof(bf16_t),
|
||||
plan.bar_ready[chunk_slot]);
|
||||
}
|
||||
if constexpr (HAS_DELTA) {
|
||||
int prefix_n = N - 1 - ns;
|
||||
if (prefix_n >= 0 && prefix_n < an) {
|
||||
cp_async_bulk(delta_buf_ptr(chunk_slot),
|
||||
delta + (long long)tb * H, H * sizeof(bf16_t),
|
||||
plan.bar_ready[chunk_slot]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
float acc32[ACC_PER_THREAD] = {};
|
||||
float eps_cache;
|
||||
asm volatile("mov.b32 %0, %1;" : "=f"(eps_cache) : "f"(rms_eps));
|
||||
|
||||
long long gci = 0;
|
||||
for (int tb = blockIdx.x; tb < TB; tb += num_ctas) {
|
||||
float m_running = -FLT_MAX;
|
||||
float s_running = 0.f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < ACC_PER_THREAD; i++) {
|
||||
acc32[i] = 0.f;
|
||||
}
|
||||
|
||||
for (int ci = 0; ci < num_chunks; ci++, gci++) {
|
||||
int ns = ci * N_CHUNK;
|
||||
int an = min(N_CHUNK, N - ns);
|
||||
int chunk_slot = (int)(gci % CHUNK_DEPTH);
|
||||
int pr = phase_of(gci);
|
||||
mbarrier_wait(plan.bar_ready[chunk_slot], pr);
|
||||
|
||||
float2 sq_local[N_CHUNK] = {};
|
||||
float2 dot_local[N_CHUNK] = {};
|
||||
|
||||
auto pass_A_body = [&](auto AN_TOK) {
|
||||
constexpr int AN = decltype(AN_TOK)::value;
|
||||
#pragma unroll
|
||||
for (int si = 0; si < SLICES_PER_GROUP; si++) {
|
||||
if constexpr (H == 7168) {
|
||||
if (si == SLICES_PER_GROUP - 1) {
|
||||
int h_base =
|
||||
6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4;
|
||||
const float* qv = &q_cache[si * VEC];
|
||||
#pragma unroll
|
||||
for (int n = 0; n < AN; n++) {
|
||||
int slot = slot_of(gci, n);
|
||||
int2 vp =
|
||||
*reinterpret_cast<const int2*>(buf_ptr(slot) + h_base);
|
||||
auto* v2 = reinterpret_cast<__nv_bfloat162*>(&vp);
|
||||
if constexpr (HAS_DELTA) {
|
||||
int prefix_n = N - 1 - ns;
|
||||
if (n == prefix_n) {
|
||||
const bf16_t* delta_ptr =
|
||||
delta_buf_ptr(chunk_slot) + h_base;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 2; j++) {
|
||||
auto delta2 = *reinterpret_cast<const __nv_bfloat162*>(
|
||||
delta_ptr + 2 * j);
|
||||
v2[j] = __hadd2(v2[j], delta2);
|
||||
}
|
||||
*reinterpret_cast<int2*>(layer_res + (long long)tb * H +
|
||||
h_base) = vp;
|
||||
}
|
||||
}
|
||||
float2 f[2] = {__bfloat1622float2(v2[0]),
|
||||
__bfloat1622float2(v2[1])};
|
||||
tmem_store<4>(my_v_tmem + (si * N_CHUNK + n) * VEC, f);
|
||||
sq_local[n] = float2_fma(f[0], f[0], sq_local[n]);
|
||||
sq_local[n] = float2_fma(f[1], f[1], sq_local[n]);
|
||||
dot_local[n] =
|
||||
float2_fma(f[0], make_float2(qv[0], qv[1]), dot_local[n]);
|
||||
dot_local[n] =
|
||||
float2_fma(f[1], make_float2(qv[2], qv[3]), dot_local[n]);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
int dt = si * CONSUMER_GROUPS + group;
|
||||
if (dt >= NHT) continue;
|
||||
int h_base = dt * K_TILE + k_local;
|
||||
const float* qv = &q_cache[si * VEC];
|
||||
|
||||
#pragma unroll
|
||||
for (int n = 0; n < AN; n++) {
|
||||
int slot = slot_of(gci, n);
|
||||
int4 vp = *reinterpret_cast<const int4*>(buf_ptr(slot) + h_base);
|
||||
auto* v2 = reinterpret_cast<__nv_bfloat162*>(&vp);
|
||||
if constexpr (HAS_DELTA) {
|
||||
int prefix_n = N - 1 - ns;
|
||||
if (n == prefix_n) {
|
||||
const bf16_t* delta_ptr = delta_buf_ptr(chunk_slot) + h_base;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC / 2; j++) {
|
||||
auto delta2 = *reinterpret_cast<const __nv_bfloat162*>(
|
||||
delta_ptr + 2 * j);
|
||||
v2[j] = __hadd2(v2[j], delta2);
|
||||
}
|
||||
*reinterpret_cast<int4*>(layer_res + (long long)tb * H +
|
||||
h_base) = vp;
|
||||
}
|
||||
}
|
||||
float2 f[4] = {
|
||||
__bfloat1622float2(v2[0]), __bfloat1622float2(v2[1]),
|
||||
__bfloat1622float2(v2[2]), __bfloat1622float2(v2[3])};
|
||||
tmem_store<VEC>(my_v_tmem + (si * N_CHUNK + n) * VEC, f);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC / 2; j++) {
|
||||
sq_local[n] = float2_fma(f[j], f[j], sq_local[n]);
|
||||
dot_local[n] = float2_fma(
|
||||
f[j], make_float2(qv[2 * j], qv[2 * j + 1]), dot_local[n]);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if constexpr (NC == 4) {
|
||||
switch (an) {
|
||||
case 4:
|
||||
pass_A_body(std::integral_constant<int, 4>{});
|
||||
break;
|
||||
case 3:
|
||||
pass_A_body(std::integral_constant<int, 3>{});
|
||||
break;
|
||||
case 2:
|
||||
pass_A_body(std::integral_constant<int, 2>{});
|
||||
break;
|
||||
case 1:
|
||||
pass_A_body(std::integral_constant<int, 1>{});
|
||||
break;
|
||||
default:
|
||||
__builtin_unreachable();
|
||||
}
|
||||
} else if constexpr (NC == 3) {
|
||||
switch (an) {
|
||||
case 3:
|
||||
pass_A_body(std::integral_constant<int, 3>{});
|
||||
break;
|
||||
case 2:
|
||||
pass_A_body(std::integral_constant<int, 2>{});
|
||||
break;
|
||||
case 1:
|
||||
pass_A_body(std::integral_constant<int, 1>{});
|
||||
break;
|
||||
default:
|
||||
__builtin_unreachable();
|
||||
}
|
||||
} else {
|
||||
static_assert(NC == 2);
|
||||
switch (an) {
|
||||
case 2:
|
||||
pass_A_body(std::integral_constant<int, 2>{});
|
||||
break;
|
||||
case 1:
|
||||
pass_A_body(std::integral_constant<int, 1>{});
|
||||
break;
|
||||
default:
|
||||
__builtin_unreachable();
|
||||
}
|
||||
}
|
||||
if (lane == 0) {
|
||||
mbarrier_arrive(plan.bar_consumed[chunk_slot]);
|
||||
}
|
||||
tmem_store_wait();
|
||||
|
||||
float2 reduce_pair[N_CHUNK];
|
||||
#pragma unroll
|
||||
for (int n = 0; n < N_CHUNK; n++) {
|
||||
reduce_pair[n] = make_float2(sq_local[n].x + sq_local[n].y,
|
||||
dot_local[n].x + dot_local[n].y);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset >>= 1) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < N_CHUNK; n++) {
|
||||
uint64_t packed = reinterpret_cast<uint64_t&>(reduce_pair[n]);
|
||||
packed = __shfl_xor_sync(0xffffffff, packed, offset);
|
||||
float2 other = reinterpret_cast<float2&>(packed);
|
||||
reduce_pair[n] = float2_add(reduce_pair[n], other);
|
||||
}
|
||||
}
|
||||
if (lane == 0) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < N_CHUNK; n++) {
|
||||
plan.ws_stats[comp_wid][n] = reduce_pair[n];
|
||||
}
|
||||
}
|
||||
named_barrier_sync(CONSUMER_THREADS, 0);
|
||||
|
||||
float local_rsig = 0.f;
|
||||
float local_logit = 0.f;
|
||||
int stat_n = lane / CONSUMER_WARPS;
|
||||
int stat_w = lane % CONSUMER_WARPS;
|
||||
float2 totals = {};
|
||||
if (stat_n < N_CHUNK) {
|
||||
totals = plan.ws_stats[stat_w][stat_n];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int offset = CONSUMER_WARPS / 2; offset > 0; offset >>= 1) {
|
||||
totals.x +=
|
||||
__shfl_down_sync(0xffffffff, totals.x, offset, CONSUMER_WARPS);
|
||||
totals.y +=
|
||||
__shfl_down_sync(0xffffffff, totals.y, offset, CONSUMER_WARPS);
|
||||
}
|
||||
if (stat_n < N_CHUNK && stat_w == 0) {
|
||||
local_rsig = rsqrtf(totals.x / H + eps_cache);
|
||||
local_logit = totals.y * local_rsig;
|
||||
}
|
||||
float logit_n[N_CHUNK];
|
||||
#pragma unroll
|
||||
for (int n = 0; n < N_CHUNK; n++) {
|
||||
logit_n[n] = __shfl_sync(0xffffffff, local_logit, n * CONSUMER_WARPS);
|
||||
}
|
||||
|
||||
float m_chunk = -FLT_MAX;
|
||||
#pragma unroll
|
||||
for (int n = 0; n < N_CHUNK; n++) {
|
||||
if (n < an) m_chunk = fmaxf(m_chunk, logit_n[n]);
|
||||
}
|
||||
float m_new = fmaxf(m_running, m_chunk);
|
||||
float corr = exp2f((m_running - m_new) * LOG2_E);
|
||||
float w_n[N_CHUNK] = {};
|
||||
float w_sum = 0.f;
|
||||
#pragma unroll
|
||||
for (int n = 0; n < N_CHUNK; n++) {
|
||||
if (n < an) {
|
||||
w_n[n] = exp2f((logit_n[n] - m_new) * LOG2_E);
|
||||
w_sum += w_n[n];
|
||||
}
|
||||
}
|
||||
|
||||
auto pass_B_body = [&](auto AN_TOK) {
|
||||
constexpr int AN = decltype(AN_TOK)::value;
|
||||
#pragma unroll
|
||||
for (int si = 0; si < SLICES_PER_GROUP; si++) {
|
||||
if constexpr (H == 7168) {
|
||||
if (si == SLICES_PER_GROUP - 1) {
|
||||
float2 corr2 = make_float2(corr, corr);
|
||||
float2 a[2];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 2; j++) {
|
||||
float2 old = make_float2(acc32[si * VEC + 2 * j],
|
||||
acc32[si * VEC + 2 * j + 1]);
|
||||
a[j] = float2_mul(old, corr2);
|
||||
}
|
||||
float2 f_cache[AN][2];
|
||||
#pragma unroll
|
||||
for (int n = 0; n < AN; n++) {
|
||||
tmem_load<4>(my_v_tmem + (si * N_CHUNK + n) * VEC,
|
||||
f_cache[n]);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int n = 0; n < AN; n++) {
|
||||
float2 wn = make_float2(w_n[n], w_n[n]);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 2; j++) {
|
||||
a[j] = float2_fma(wn, f_cache[n][j], a[j]);
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 2; j++) {
|
||||
acc32[si * VEC + 2 * j] = a[j].x;
|
||||
acc32[si * VEC + 2 * j + 1] = a[j].y;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
int dt = si * CONSUMER_GROUPS + group;
|
||||
if (dt >= NHT) continue;
|
||||
float2 corr2 = make_float2(corr, corr);
|
||||
float2 a[VEC / 2];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC / 2; j++) {
|
||||
float2 old = make_float2(acc32[si * VEC + 2 * j],
|
||||
acc32[si * VEC + 2 * j + 1]);
|
||||
a[j] = float2_mul(old, corr2);
|
||||
}
|
||||
float2 f_cache[AN][VEC / 2];
|
||||
#pragma unroll
|
||||
for (int n = 0; n < AN; n++) {
|
||||
tmem_load<VEC>(my_v_tmem + (si * N_CHUNK + n) * VEC, f_cache[n]);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int n = 0; n < AN; n++) {
|
||||
float2 wn = make_float2(w_n[n], w_n[n]);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC / 2; j++) {
|
||||
a[j] = float2_fma(wn, f_cache[n][j], a[j]);
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC / 2; j++) {
|
||||
acc32[si * VEC + 2 * j] = a[j].x;
|
||||
acc32[si * VEC + 2 * j + 1] = a[j].y;
|
||||
}
|
||||
}
|
||||
};
|
||||
if constexpr (NC == 4) {
|
||||
switch (an) {
|
||||
case 4:
|
||||
pass_B_body(std::integral_constant<int, 4>{});
|
||||
break;
|
||||
case 3:
|
||||
pass_B_body(std::integral_constant<int, 3>{});
|
||||
break;
|
||||
case 2:
|
||||
pass_B_body(std::integral_constant<int, 2>{});
|
||||
break;
|
||||
case 1:
|
||||
pass_B_body(std::integral_constant<int, 1>{});
|
||||
break;
|
||||
default:
|
||||
__builtin_unreachable();
|
||||
}
|
||||
} else if constexpr (NC == 3) {
|
||||
switch (an) {
|
||||
case 3:
|
||||
pass_B_body(std::integral_constant<int, 3>{});
|
||||
break;
|
||||
case 2:
|
||||
pass_B_body(std::integral_constant<int, 2>{});
|
||||
break;
|
||||
case 1:
|
||||
pass_B_body(std::integral_constant<int, 1>{});
|
||||
break;
|
||||
default:
|
||||
__builtin_unreachable();
|
||||
}
|
||||
} else {
|
||||
static_assert(NC == 2);
|
||||
switch (an) {
|
||||
case 2:
|
||||
pass_B_body(std::integral_constant<int, 2>{});
|
||||
break;
|
||||
case 1:
|
||||
pass_B_body(std::integral_constant<int, 1>{});
|
||||
break;
|
||||
default:
|
||||
__builtin_unreachable();
|
||||
}
|
||||
}
|
||||
|
||||
s_running = s_running * corr + w_sum;
|
||||
m_running = m_new;
|
||||
}
|
||||
|
||||
float inv_s = 1.f / s_running;
|
||||
bf16_t* out_ptr = output + (long long)tb * H;
|
||||
float2 output_sq_pair = {};
|
||||
// When output RMSNorm is fused, the softmax denominator cancels:
|
||||
// (acc / s) * rsqrt(mean((acc / s)^2) + eps)
|
||||
// = acc * rsqrt(mean(acc^2) + eps * s^2).
|
||||
#pragma unroll
|
||||
for (int si = 0; si < SLICES_PER_GROUP; si++) {
|
||||
if constexpr (H == 7168) {
|
||||
if (si == SLICES_PER_GROUP - 1) {
|
||||
int h_base = 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4;
|
||||
uint2 packed;
|
||||
auto* ov2 = reinterpret_cast<__nv_bfloat162*>(&packed);
|
||||
float2 inv2 = make_float2(inv_s, inv_s);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 2; j++) {
|
||||
float2 old = make_float2(acc32[si * VEC + 2 * j],
|
||||
acc32[si * VEC + 2 * j + 1]);
|
||||
if constexpr (HAS_OUTPUT_NORM) {
|
||||
output_sq_pair = float2_fma(old, old, output_sq_pair);
|
||||
} else {
|
||||
float2 mixed = float2_mul(old, inv2);
|
||||
ov2[j] = __float22bfloat162_rn(mixed);
|
||||
}
|
||||
}
|
||||
if constexpr (!HAS_OUTPUT_NORM) {
|
||||
*reinterpret_cast<uint2*>(out_ptr + h_base) = packed;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
int dt = si * CONSUMER_GROUPS + group;
|
||||
if (dt >= NHT) continue;
|
||||
int h_base = dt * K_TILE + k_local;
|
||||
uint4 packed;
|
||||
auto* ov2 = reinterpret_cast<__nv_bfloat162*>(&packed);
|
||||
float2 inv2 = make_float2(inv_s, inv_s);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC / 2; j++) {
|
||||
float2 old =
|
||||
make_float2(acc32[si * VEC + 2 * j], acc32[si * VEC + 2 * j + 1]);
|
||||
if constexpr (HAS_OUTPUT_NORM) {
|
||||
output_sq_pair = float2_fma(old, old, output_sq_pair);
|
||||
} else {
|
||||
float2 mixed = float2_mul(old, inv2);
|
||||
ov2[j] = __float22bfloat162_rn(mixed);
|
||||
}
|
||||
}
|
||||
if constexpr (!HAS_OUTPUT_NORM) {
|
||||
*reinterpret_cast<uint4*>(out_ptr + h_base) = packed;
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (HAS_OUTPUT_NORM) {
|
||||
if constexpr (OUTPUT_NORM_IN_SMEM) {
|
||||
// The immutable weight copy is acquired once, at its first use.
|
||||
if (tb == blockIdx.x) {
|
||||
mbarrier_wait(plan.bar_output_norm_ready, 0);
|
||||
}
|
||||
}
|
||||
float output_sq = output_sq_pair.x + output_sq_pair.y;
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset >>= 1) {
|
||||
output_sq += __shfl_xor_sync(0xffffffff, output_sq, offset);
|
||||
}
|
||||
if (lane == 0) {
|
||||
plan.ws_stats[comp_wid][0] = make_float2(output_sq, 0.f);
|
||||
}
|
||||
named_barrier_sync(CONSUMER_THREADS, 0);
|
||||
float total_sq = lane < CONSUMER_WARPS ? plan.ws_stats[lane][0].x : 0.f;
|
||||
#pragma unroll
|
||||
for (int offset = CONSUMER_WARPS / 2; offset > 0; offset >>= 1) {
|
||||
total_sq +=
|
||||
__shfl_down_sync(0xffffffff, total_sq, offset, CONSUMER_WARPS);
|
||||
}
|
||||
if (lane == 0) {
|
||||
total_sq =
|
||||
rsqrtf(total_sq / H + output_norm_eps * s_running * s_running);
|
||||
}
|
||||
float output_rsigma = __shfl_sync(0xffffffff, total_sq, 0);
|
||||
#pragma unroll
|
||||
for (int si = 0; si < SLICES_PER_GROUP; si++) {
|
||||
if constexpr (H == 7168) {
|
||||
if (si == SLICES_PER_GROUP - 1) {
|
||||
int h_base = 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4;
|
||||
uint2 packed;
|
||||
auto* values = reinterpret_cast<bf16_t*>(&packed);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 4; j++) {
|
||||
const bf16_t* weight_ptr =
|
||||
OUTPUT_NORM_IN_SMEM ? output_norm_buf : output_norm_weight;
|
||||
float weight = __bfloat162float(weight_ptr[h_base + j]);
|
||||
values[j] = __float2bfloat16(acc32[si * VEC + j] *
|
||||
output_rsigma * weight);
|
||||
}
|
||||
*reinterpret_cast<uint2*>(out_ptr + h_base) = packed;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
int dt = si * CONSUMER_GROUPS + group;
|
||||
if (dt >= NHT) continue;
|
||||
int h_base = dt * K_TILE + k_local;
|
||||
uint4 packed;
|
||||
auto* values = reinterpret_cast<bf16_t*>(&packed);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC; j++) {
|
||||
const bf16_t* weight_ptr =
|
||||
OUTPUT_NORM_IN_SMEM ? output_norm_buf : output_norm_weight;
|
||||
float weight = __bfloat162float(weight_ptr[h_base + j]);
|
||||
values[j] =
|
||||
__float2bfloat16(acc32[si * VEC + j] * output_rsigma * weight);
|
||||
}
|
||||
*reinterpret_cast<uint4*>(out_ptr + h_base) = packed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
__syncthreads();
|
||||
if (wid == 1) {
|
||||
tmem_free(plan.tmem_base, TMEM_COLS_ALLOC);
|
||||
}
|
||||
#else
|
||||
if (threadIdx.x == 0) {
|
||||
printf("attn_res_fwd_online_v2_kernel requires sm_10x\n");
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
template <int H, int NC = N_CHUNK_DEFAULT, bool RELEASE_TMEM = false,
|
||||
bool HAS_DELTA = false, bool HAS_OUTPUT_NORM = false,
|
||||
bool OUTPUT_NORM_IN_SMEM = false>
|
||||
static void launch_fwd(const bf16_t* block_residual, bf16_t* layer_residual,
|
||||
const bf16_t* delta, const bf16_t* res_weight,
|
||||
const bf16_t* rms_weight, bf16_t* output, int N, int T,
|
||||
int B, float rms_eps, int num_sm, cudaStream_t stream,
|
||||
const bf16_t* output_norm_weight = nullptr,
|
||||
float output_norm_eps = 0.f, int block_stride_m = 0,
|
||||
int block_stride_r = 0) {
|
||||
constexpr size_t smem_size =
|
||||
((size_t)CHUNK_DEPTH * (NC + (HAS_DELTA ? 1 : 0)) * H * sizeof(bf16_t) +
|
||||
(OUTPUT_NORM_IN_SMEM ? (size_t)H * sizeof(bf16_t) : 0) +
|
||||
sizeof(FwdSmemPlan<NC>) + 15) &
|
||||
~size_t(15);
|
||||
auto kernel =
|
||||
&attn_res_fwd_online_v2_kernel<H, NC, RELEASE_TMEM, HAS_DELTA,
|
||||
HAS_OUTPUT_NORM, OUTPUT_NORM_IN_SMEM>;
|
||||
static bool attrs_set = false;
|
||||
if (!attrs_set) {
|
||||
if (smem_size > 48 * 1024) {
|
||||
cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
smem_size);
|
||||
}
|
||||
attrs_set = true;
|
||||
}
|
||||
int grid = RELEASE_TMEM ? num_sm * 2 : num_sm;
|
||||
cudaLaunchConfig_t config{};
|
||||
config.gridDim = grid;
|
||||
config.blockDim = BLK;
|
||||
config.dynamicSmemBytes = smem_size;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
config.attrs = attrs;
|
||||
config.numAttrs = 1;
|
||||
cudaLaunchKernelEx(&config, kernel, block_residual, layer_residual, delta,
|
||||
res_weight, rms_weight, output, N, T, B, block_stride_m,
|
||||
block_stride_r, rms_eps, output_norm_weight,
|
||||
output_norm_eps);
|
||||
}
|
||||
|
||||
} // namespace fwd_prod_v2
|
||||
} // namespace sm100
|
||||
|
||||
void kimi_k3_attn_res(torch::stable::Tensor& prefix,
|
||||
torch::stable::Tensor const& delta,
|
||||
torch::stable::Tensor const& blocks,
|
||||
torch::stable::Tensor const& norm_weight,
|
||||
torch::stable::Tensor const& qk_weight,
|
||||
torch::stable::Tensor const& output_norm_weight,
|
||||
torch::stable::Tensor& output, int64_t num_blocks,
|
||||
double eps, double output_norm_eps) {
|
||||
int const num_tokens = static_cast<int>(prefix.size(0));
|
||||
int const device = prefix.get_device_index();
|
||||
torch::stable::accelerator::DeviceGuard const device_guard(device);
|
||||
cudaDeviceProp const* properties = get_device_prop();
|
||||
STD_TORCH_CHECK(properties->major == 10,
|
||||
"Kimi K3 AttnRes requires the SM100 family");
|
||||
|
||||
using namespace sm100::fwd_prod_v2;
|
||||
// Two-source chunks and two resident CTAs are beneficial once setup is
|
||||
// amortized by the long, full eight-block prefill workload.
|
||||
if (num_blocks == 8 && num_tokens >= 4096) {
|
||||
launch_fwd<7168, 2, true, true, true, true>(
|
||||
static_cast<bf16_t const*>(blocks.data_ptr()),
|
||||
static_cast<bf16_t*>(prefix.data_ptr()),
|
||||
static_cast<bf16_t const*>(delta.data_ptr()),
|
||||
static_cast<bf16_t const*>(qk_weight.data_ptr()),
|
||||
static_cast<bf16_t const*>(norm_weight.data_ptr()),
|
||||
static_cast<bf16_t*>(output.data_ptr()),
|
||||
static_cast<int>(num_blocks) + 1, num_tokens, 1,
|
||||
static_cast<float>(eps), properties->multiProcessorCount,
|
||||
get_current_cuda_stream(device),
|
||||
static_cast<bf16_t const*>(output_norm_weight.data_ptr()),
|
||||
static_cast<float>(output_norm_eps), static_cast<int>(blocks.stride(0)),
|
||||
static_cast<int>(blocks.stride(1)));
|
||||
} else {
|
||||
launch_fwd<7168, 4, false, true, true, true>(
|
||||
static_cast<bf16_t const*>(blocks.data_ptr()),
|
||||
static_cast<bf16_t*>(prefix.data_ptr()),
|
||||
static_cast<bf16_t const*>(delta.data_ptr()),
|
||||
static_cast<bf16_t const*>(qk_weight.data_ptr()),
|
||||
static_cast<bf16_t const*>(norm_weight.data_ptr()),
|
||||
static_cast<bf16_t*>(output.data_ptr()),
|
||||
static_cast<int>(num_blocks) + 1, num_tokens, 1,
|
||||
static_cast<float>(eps), properties->multiProcessorCount,
|
||||
get_current_cuda_stream(device),
|
||||
static_cast<bf16_t const*>(output_norm_weight.data_ptr()),
|
||||
static_cast<float>(output_norm_eps), static_cast<int>(blocks.stride(0)),
|
||||
static_cast<int>(blocks.stride(1)));
|
||||
}
|
||||
cudaError_t const error = cudaGetLastError();
|
||||
STD_TORCH_CHECK(
|
||||
error == cudaSuccess,
|
||||
"Kimi K3 AttnRes kernel launch failed: ", cudaGetErrorString(error));
|
||||
}
|
||||
@@ -315,6 +315,17 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
std::optional<torch::stable::Tensor> index_q_out,
|
||||
const std::string& kv_cache_dtype, bool skip_index_branch);
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
void kimi_k3_attn_res(torch::stable::Tensor& prefix,
|
||||
torch::stable::Tensor const& delta,
|
||||
torch::stable::Tensor const& blocks,
|
||||
torch::stable::Tensor const& norm_weight,
|
||||
torch::stable::Tensor const& qk_weight,
|
||||
torch::stable::Tensor const& output_norm_weight,
|
||||
torch::stable::Tensor& output, int64_t num_blocks,
|
||||
double eps, double output_norm_eps);
|
||||
#endif
|
||||
|
||||
// Sampler kernels (shared CUDA/ROCm)
|
||||
void apply_repetition_penalties_(
|
||||
torch::stable::Tensor& logits, const torch::stable::Tensor& prompt_mask,
|
||||
|
||||
@@ -468,6 +468,14 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"int block_size, Tensor!? q_out, Tensor!? index_q_out, "
|
||||
"str kv_cache_dtype, bool skip_index_branch=False) -> ()");
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.def(
|
||||
"kimi_k3_attn_res("
|
||||
"Tensor! prefix, Tensor delta, Tensor blocks, Tensor norm_weight, "
|
||||
"Tensor qk_weight, Tensor output_norm_weight, Tensor! output, "
|
||||
"int num_blocks, float eps, float output_norm_eps) -> ()");
|
||||
#endif
|
||||
|
||||
// Apply repetition penalties to logits in-place.
|
||||
ops.def(
|
||||
"apply_repetition_penalties_(Tensor! logits, Tensor prompt_mask, "
|
||||
@@ -693,6 +701,9 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
#endif
|
||||
ops.impl("fused_minimax_m3_qknorm_rope_kv_insert",
|
||||
TORCH_BOX(&fused_minimax_m3_qknorm_rope_kv_insert));
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res));
|
||||
#endif
|
||||
|
||||
// Sampler kernels (shared CUDA/ROCm)
|
||||
ops.impl("apply_repetition_penalties_",
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AMD AttnRes requires ROCm",
|
||||
)
|
||||
|
||||
|
||||
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
|
||||
storage = torch.randn(
|
||||
*shape[:-1],
|
||||
shape[-1] + padding,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
return storage[..., : shape[-1]]
|
||||
|
||||
|
||||
def _reference(
|
||||
prefix: torch.Tensor,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
num_blocks: int,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
hidden_size = prefix.shape[-1]
|
||||
values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1)
|
||||
keys = F.rms_norm(values, (hidden_size,), norm_weight, eps)
|
||||
probs = (keys @ qk_weight).softmax(dim=-1)
|
||||
return torch.matmul(probs.unsqueeze(1), values).squeeze(1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"num_tokens",
|
||||
"num_blocks",
|
||||
"block_capacity",
|
||||
"hidden_size",
|
||||
"row_padding",
|
||||
),
|
||||
[
|
||||
pytest.param(0, 3, 5, 128, 0, id="empty"),
|
||||
pytest.param(1, 1, 2, 128, 0, id="decode-single"),
|
||||
pytest.param(17, 4, 6, 1024, 7, id="decode-padded"),
|
||||
pytest.param(320, 8, 10, 7168, 0, id="prefill-full"),
|
||||
],
|
||||
)
|
||||
def test_amd_attn_res_matches_reference(
|
||||
num_tokens: int,
|
||||
num_blocks: int,
|
||||
block_capacity: int,
|
||||
hidden_size: int,
|
||||
row_padding: int,
|
||||
) -> None:
|
||||
eps = 1e-5
|
||||
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=row_padding)
|
||||
blocks = _randn_with_row_padding(
|
||||
num_tokens,
|
||||
block_capacity,
|
||||
hidden_size,
|
||||
padding=row_padding,
|
||||
)
|
||||
norm_weight = 1 + 0.1 * torch.randn(
|
||||
hidden_size, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
qk_weight = (
|
||||
torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5
|
||||
)
|
||||
expected = _reference(
|
||||
prefix,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
num_blocks,
|
||||
eps,
|
||||
)
|
||||
original_prefix = prefix.clone()
|
||||
original_blocks = blocks.clone()
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
num_blocks,
|
||||
eps,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(prefix, original_prefix, atol=0, rtol=0)
|
||||
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
|
||||
assert actual.shape == prefix.shape
|
||||
assert actual.is_contiguous()
|
||||
@@ -0,0 +1,193 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops import attn_res
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
HIDDEN_SIZE = 7168
|
||||
MAX_BLOCKS = 8
|
||||
EPS = 1e-5
|
||||
|
||||
|
||||
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
|
||||
storage = torch.randn(
|
||||
*shape[:-1],
|
||||
shape[-1] + padding,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
return storage[..., : shape[-1]]
|
||||
|
||||
|
||||
def _reference(
|
||||
prefix: torch.Tensor,
|
||||
delta: torch.Tensor | None,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
output_norm_weight: torch.Tensor | None,
|
||||
num_blocks: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if delta is not None:
|
||||
prefix = prefix + delta
|
||||
values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1)
|
||||
keys = F.rms_norm(values, (HIDDEN_SIZE,), norm_weight, EPS)
|
||||
probs = (keys @ qk_weight).softmax(dim=-1)
|
||||
output = torch.matmul(probs.unsqueeze(1), values).squeeze(1)
|
||||
if output_norm_weight is not None:
|
||||
output = F.rms_norm(output, (HIDDEN_SIZE,), output_norm_weight, EPS)
|
||||
return output, prefix
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"num_tokens",
|
||||
"num_blocks",
|
||||
"row_padding",
|
||||
"write_block",
|
||||
"has_delta",
|
||||
"backend",
|
||||
),
|
||||
[
|
||||
pytest.param(1, 0, 0, True, False, "triton", id="triton-empty"),
|
||||
pytest.param(1, 0, 0, True, True, "triton", id="triton-empty-add"),
|
||||
pytest.param(17, 5, 7, True, False, "triton", id="triton-write"),
|
||||
pytest.param(17, 5, 7, True, True, "triton", id="triton-write-add"),
|
||||
pytest.param(3, 8, 0, False, False, "triton", id="triton-full"),
|
||||
pytest.param(3, 8, 0, False, True, "triton", id="triton-full-add"),
|
||||
pytest.param(320, 1, 0, False, True, "nvidia", id="nvidia-1"),
|
||||
pytest.param(320, 4, 0, False, True, "nvidia", id="nvidia-4"),
|
||||
pytest.param(320, 8, 0, False, True, "nvidia", id="nvidia-8"),
|
||||
],
|
||||
)
|
||||
def test_attn_res(
|
||||
num_tokens: int,
|
||||
num_blocks: int,
|
||||
row_padding: int,
|
||||
write_block: bool,
|
||||
has_delta: bool,
|
||||
backend: str,
|
||||
):
|
||||
if backend == "nvidia" and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("NVIDIA AttnRes requires the SM100 family")
|
||||
|
||||
prefix = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
|
||||
delta = (
|
||||
_randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
|
||||
if has_delta
|
||||
else None
|
||||
)
|
||||
blocks = _randn_with_row_padding(
|
||||
num_tokens, MAX_BLOCKS, HIDDEN_SIZE, padding=row_padding
|
||||
)
|
||||
norm_weight = 1 + 0.1 * torch.randn(
|
||||
HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
qk_weight = (
|
||||
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
|
||||
)
|
||||
output_norm_weight = 1 + 0.1 * torch.randn(
|
||||
HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
original_blocks = blocks.clone()
|
||||
expected, expected_prefix = _reference(
|
||||
prefix.clone(),
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
)
|
||||
block_write_idx = num_blocks if write_block else -1
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
block_write_idx,
|
||||
EPS,
|
||||
EPS,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0)
|
||||
if write_block:
|
||||
original_blocks[:, block_write_idx].copy_(expected_prefix)
|
||||
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_blocks", range(MAX_BLOCKS + 1))
|
||||
def test_attn_res_block_counts(num_blocks: int):
|
||||
prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
blocks = torch.randn(
|
||||
1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
norm_weight = torch.ones(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
qk_weight = (
|
||||
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
|
||||
)
|
||||
output_norm_weight = torch.ones_like(norm_weight)
|
||||
expected, _ = _reference(
|
||||
prefix.clone(),
|
||||
None,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
)
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
None,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
-1,
|
||||
EPS,
|
||||
EPS,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def test_attn_res_without_output_norm():
|
||||
prefix = torch.randn(7, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
delta = torch.randn_like(prefix)
|
||||
blocks = torch.randn(
|
||||
7, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
norm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
qk_weight = (
|
||||
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
|
||||
)
|
||||
expected, _ = _reference(
|
||||
prefix.clone(), delta, blocks, norm_weight, qk_weight, None, MAX_BLOCKS
|
||||
)
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
None,
|
||||
MAX_BLOCKS,
|
||||
-1,
|
||||
EPS,
|
||||
0.0,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
@@ -2746,6 +2746,33 @@ def concat_and_cache_mla(
|
||||
)
|
||||
|
||||
|
||||
def kimi_k3_attn_res(
|
||||
prefix: torch.Tensor,
|
||||
delta: torch.Tensor,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
output_norm_weight: torch.Tensor,
|
||||
num_blocks: int,
|
||||
eps: float,
|
||||
output_norm_eps: float,
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty_like(prefix)
|
||||
torch.ops._C.kimi_k3_attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
output,
|
||||
num_blocks,
|
||||
eps,
|
||||
output_norm_eps,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def concat_and_cache_mla_rope_fused(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,132 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code adapted from the flash-linear-attention project.
|
||||
# The original source code was licensed under the MIT license and included
|
||||
# the following copyright notice:
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _attn_res_kernel(
|
||||
prefix_ptr,
|
||||
blocks_ptr,
|
||||
norm_weight_ptr,
|
||||
qk_weight_ptr,
|
||||
output_ptr,
|
||||
stride_prefix_m: tl.constexpr,
|
||||
stride_block_m: tl.constexpr,
|
||||
stride_block_r: tl.constexpr,
|
||||
stride_output_m: tl.constexpr,
|
||||
num_blocks: tl.constexpr,
|
||||
hidden_size: tl.constexpr,
|
||||
eps: tl.constexpr,
|
||||
BLOCK_L: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
row_idx = tl.program_id(0).to(tl.int64)
|
||||
d_offsets = tl.max_contiguous(tl.arange(0, BLOCK_D), BLOCK_D)
|
||||
d_mask = d_offsets < hidden_size
|
||||
|
||||
prefix = tl.load(
|
||||
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
|
||||
mask=d_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
input_qk_weight = tl.load(norm_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(
|
||||
tl.float32
|
||||
) * tl.load(qk_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(tl.float32)
|
||||
|
||||
max_logit = tl.full((), -float("inf"), tl.float32)
|
||||
denominator = tl.zeros((), tl.float32)
|
||||
mixed = tl.zeros((BLOCK_D,), tl.float32)
|
||||
num_sources = num_blocks + 1
|
||||
|
||||
for source_tile in range(tl.cdiv(num_sources, BLOCK_L)):
|
||||
source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L)
|
||||
source_mask = source_offsets < num_sources
|
||||
is_prefix = source_offsets == num_blocks
|
||||
block_ptrs = (
|
||||
blocks_ptr
|
||||
+ row_idx * stride_block_m
|
||||
+ source_offsets[:, None] * stride_block_r
|
||||
+ d_offsets[None, :]
|
||||
)
|
||||
block_values = tl.load(
|
||||
block_ptrs,
|
||||
mask=(source_mask[:, None] & ~is_prefix[:, None] & d_mask[None, :]),
|
||||
other=0.0,
|
||||
eviction_policy="evict_first",
|
||||
).to(tl.float32)
|
||||
values = tl.where(is_prefix[:, None], prefix[None, :], block_values)
|
||||
reciprocal_std = tl.rsqrt(
|
||||
tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps
|
||||
)
|
||||
logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std
|
||||
scores = tl.where(source_mask, logits, -float("inf"))
|
||||
|
||||
new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0))
|
||||
old_scale = tl.exp(max_logit - new_max_logit)
|
||||
block_scales = tl.exp(scores - new_max_logit)
|
||||
denominator = denominator * old_scale + tl.sum(block_scales, axis=0)
|
||||
mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0)
|
||||
max_logit = new_max_logit
|
||||
|
||||
output = mixed / denominator
|
||||
tl.store(
|
||||
output_ptr + row_idx * stride_output_m + d_offsets,
|
||||
output,
|
||||
mask=d_mask,
|
||||
)
|
||||
|
||||
|
||||
def attn_res(
|
||||
prefix: torch.Tensor,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
num_blocks: int,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, hidden_size = prefix.shape
|
||||
assert 0 < num_blocks <= blocks.shape[1]
|
||||
assert blocks.shape[0] == num_tokens
|
||||
assert norm_weight.numel() == hidden_size
|
||||
assert qk_weight.numel() == hidden_size
|
||||
assert prefix.stride(-1) == 1
|
||||
assert blocks.stride(-1) == 1
|
||||
assert norm_weight.stride(-1) == 1
|
||||
assert qk_weight.stride(-1) == 1
|
||||
|
||||
output = prefix.new_empty(prefix.shape)
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
|
||||
if num_tokens >= 256 or num_blocks <= 1:
|
||||
block_l, num_warps = 1, 4
|
||||
else:
|
||||
block_l, num_warps = 4, 8
|
||||
_attn_res_kernel[(num_tokens,)](
|
||||
prefix,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output,
|
||||
prefix.stride(0),
|
||||
blocks.stride(0),
|
||||
blocks.stride(1),
|
||||
output.stride(0),
|
||||
num_blocks,
|
||||
hidden_size,
|
||||
eps,
|
||||
BLOCK_L=block_l,
|
||||
BLOCK_D=triton.next_power_of_2(hidden_size),
|
||||
num_warps=num_warps,
|
||||
num_stages=2,
|
||||
)
|
||||
return output
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .attn_res import attn_res
|
||||
|
||||
__all__ = ["attn_res"]
|
||||
@@ -0,0 +1,245 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code adapted from the flash-linear-attention project.
|
||||
# The original source code was licensed under the MIT license and included
|
||||
# the following copyright notice:
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
|
||||
def get_attn_res_triton_warmup_profiles(
|
||||
max_blocks: int,
|
||||
) -> tuple[tuple[int, bool, int, bool], ...]:
|
||||
"""Return the small-batch profiles that bypass the native kernel."""
|
||||
profiles = [
|
||||
(num_blocks, False, -1, True) for num_blocks in range(2, max_blocks + 1)
|
||||
]
|
||||
profiles.extend(
|
||||
(block_write_idx, True, block_write_idx, True)
|
||||
for block_write_idx in range(2, max_blocks)
|
||||
)
|
||||
profiles.append((max_blocks, True, -1, False))
|
||||
return tuple(profiles)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _attn_res_kernel(
|
||||
prefix_ptr,
|
||||
delta_ptr,
|
||||
blocks_ptr,
|
||||
norm_weight_ptr,
|
||||
qk_weight_ptr,
|
||||
output_norm_weight_ptr,
|
||||
output_ptr,
|
||||
stride_prefix_m: tl.constexpr,
|
||||
stride_delta_m: tl.constexpr,
|
||||
stride_block_m: tl.constexpr,
|
||||
stride_block_r: tl.constexpr,
|
||||
stride_output_m: tl.constexpr,
|
||||
num_blocks: tl.constexpr,
|
||||
hidden_size: tl.constexpr,
|
||||
block_write_idx: tl.constexpr,
|
||||
eps: tl.constexpr,
|
||||
output_norm_eps: tl.constexpr,
|
||||
HAS_DELTA: tl.constexpr,
|
||||
WRITE_BLOCK: tl.constexpr,
|
||||
APPLY_OUTPUT_NORM: tl.constexpr,
|
||||
BLOCK_L: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
row_idx = tl.program_id(0).to(tl.int64)
|
||||
d_offsets = tl.max_contiguous(tl.arange(0, BLOCK_D), BLOCK_D)
|
||||
d_mask = d_offsets < hidden_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
updated_prefix = tl.load(
|
||||
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
|
||||
mask=d_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
if HAS_DELTA:
|
||||
delta = tl.load(
|
||||
delta_ptr + row_idx * stride_delta_m + d_offsets,
|
||||
mask=d_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
updated_prefix += delta
|
||||
# Match the BF16 prefix-add result before using it as a residual source.
|
||||
updated_prefix = updated_prefix.to(prefix_ptr.dtype.element_ty).to(tl.float32)
|
||||
tl.store(
|
||||
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
|
||||
updated_prefix,
|
||||
mask=d_mask,
|
||||
)
|
||||
if WRITE_BLOCK:
|
||||
tl.store(
|
||||
blocks_ptr
|
||||
+ row_idx * stride_block_m
|
||||
+ block_write_idx * stride_block_r
|
||||
+ d_offsets,
|
||||
updated_prefix,
|
||||
mask=d_mask,
|
||||
)
|
||||
# With only the prefix source, the AttnRes softmax is exactly one.
|
||||
if num_blocks == 0:
|
||||
mixed = updated_prefix
|
||||
else:
|
||||
# Reloading avoids keeping the full prefix vector live across the loop.
|
||||
if HAS_DELTA:
|
||||
tl.debug_barrier()
|
||||
input_qk_weight = tl.load(
|
||||
norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
|
||||
).to(tl.float32) * tl.load(
|
||||
qk_weight_ptr + d_offsets, mask=d_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
max_logit = tl.full((), -float("inf"), tl.float32)
|
||||
denominator = tl.zeros((), tl.float32)
|
||||
mixed = tl.zeros((BLOCK_D,), tl.float32)
|
||||
|
||||
num_sources = num_blocks + 1
|
||||
for source_tile in range(tl.cdiv(num_sources, BLOCK_L)):
|
||||
source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L)
|
||||
source_mask = source_offsets < num_sources
|
||||
is_prefix = source_offsets == num_blocks
|
||||
block_ptrs = (
|
||||
blocks_ptr
|
||||
+ row_idx * stride_block_m
|
||||
+ source_offsets[:, None] * stride_block_r
|
||||
+ d_offsets[None, :]
|
||||
)
|
||||
prefix_ptrs = (
|
||||
prefix_ptr
|
||||
+ row_idx * stride_prefix_m
|
||||
+ source_offsets[:, None] * 0
|
||||
+ d_offsets[None, :]
|
||||
)
|
||||
value_ptrs = tl.where(is_prefix[:, None], prefix_ptrs, block_ptrs)
|
||||
values = tl.load(
|
||||
value_ptrs,
|
||||
mask=source_mask[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
eviction_policy="evict_first",
|
||||
).to(tl.float32)
|
||||
reciprocal_std = tl.rsqrt(
|
||||
tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps
|
||||
)
|
||||
logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std
|
||||
scores = tl.where(source_mask, logits, -float("inf"))
|
||||
|
||||
new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0))
|
||||
old_scale = tl.exp(max_logit - new_max_logit)
|
||||
block_scales = tl.exp(scores - new_max_logit)
|
||||
denominator = denominator * old_scale + tl.sum(block_scales, axis=0)
|
||||
mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0)
|
||||
max_logit = new_max_logit
|
||||
|
||||
mixed /= denominator
|
||||
output = mixed
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
if APPLY_OUTPUT_NORM:
|
||||
output_reciprocal_std = tl.rsqrt(
|
||||
tl.sum(tl.where(d_mask, mixed * mixed, 0.0), axis=0) * (1.0 / hidden_size)
|
||||
+ output_norm_eps
|
||||
)
|
||||
output_norm_weight = tl.load(
|
||||
output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
output = mixed * output_reciprocal_std * output_norm_weight
|
||||
tl.store(
|
||||
output_ptr + row_idx * stride_output_m + d_offsets,
|
||||
output,
|
||||
mask=d_mask,
|
||||
)
|
||||
|
||||
|
||||
def attn_res(
|
||||
prefix: torch.Tensor,
|
||||
delta: torch.Tensor | None,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
output_norm_weight: torch.Tensor | None,
|
||||
num_blocks: int,
|
||||
block_write_idx: int,
|
||||
eps: float,
|
||||
output_norm_eps: float,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, hidden_size = prefix.shape
|
||||
assert prefix.stride(-1) == 1
|
||||
assert delta is None or delta.stride(-1) == 1
|
||||
assert blocks.stride(-1) == 1
|
||||
assert norm_weight.stride(-1) == 1
|
||||
assert qk_weight.stride(-1) == 1
|
||||
assert output_norm_weight is None or output_norm_weight.stride(-1) == 1
|
||||
# The in-tree NVIDIA kernel covers the common fused-add + output-norm path;
|
||||
# Triton handles block boundaries and final pre-norm output.
|
||||
if (
|
||||
hidden_size == 7168
|
||||
and delta is not None
|
||||
and output_norm_weight is not None
|
||||
and num_blocks > 0
|
||||
and block_write_idx < 0
|
||||
and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
return ops.kimi_k3_attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
eps,
|
||||
output_norm_eps,
|
||||
)
|
||||
output = prefix.new_empty(prefix.shape)
|
||||
# Tuned on GB300: source tiling helps decode, while one-source tiles scale
|
||||
# better for prefill.
|
||||
# Keep get_attn_res_triton_warmup_profiles in sync with these fallbacks.
|
||||
if num_tokens >= 256 or num_blocks <= 1:
|
||||
block_l, num_warps = 1, 4
|
||||
else:
|
||||
block_l, num_warps = 4, 8
|
||||
_attn_res_kernel[(num_tokens,)](
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
output,
|
||||
prefix.stride(0),
|
||||
0 if delta is None else delta.stride(0),
|
||||
blocks.stride(0),
|
||||
blocks.stride(1),
|
||||
output.stride(0),
|
||||
num_blocks,
|
||||
hidden_size,
|
||||
block_write_idx,
|
||||
eps,
|
||||
output_norm_eps,
|
||||
HAS_DELTA=delta is not None,
|
||||
WRITE_BLOCK=block_write_idx >= 0,
|
||||
APPLY_OUTPUT_NORM=output_norm_weight is not None,
|
||||
BLOCK_L=block_l,
|
||||
BLOCK_D=triton.next_power_of_2(hidden_size),
|
||||
num_warps=num_warps,
|
||||
num_stages=2,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return output
|
||||
Reference in New Issue
Block a user