diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index af90308d6c..a7c7aa9022 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -61,6 +61,18 @@ steps: # FA4 kernel tests require SM100; the suite skips them elsewhere. - pytest -v -s models/inkling +- label: Kimi K3 Unit Tests (B200) + key: kimi-k3-unit-tests-b200 + timeout_in_minutes: 40 + device: b200-k8s + source_file_dependencies: + - vllm/models/kimi_k3/ + - csrc/libtorch_stable/kimi_k3/ + - tests/models/kimi_k3/ + commands: + # The native NVIDIA AttnRes kernel requires the SM100 family. + - pytest -v -s models/kimi_k3 + - label: Basic Models Test (Other CPU) # 5min key: basic-models-test-other-cpu depends_on: diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a4e23ad0b..cdda81ea46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1079,6 +1079,24 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") set(MLA_ARCHS) endif() + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) + cuda_archs_loose_intersection(KIMI_K3_ATTN_RES_ARCHS + "10.0f" "${CUDA_ARCHS}") + endif() + if(KIMI_K3_ATTN_RES_ARCHS) + set(KIMI_K3_ATTN_RES_SRC + "csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu") + set_gencode_flags_for_srcs( + SRCS "${KIMI_K3_ATTN_RES_SRC}" + CUDA_ARCHS "${KIMI_K3_ATTN_RES_ARCHS}") + set_property(SOURCE ${KIMI_K3_ATTN_RES_SRC} APPEND PROPERTY + COMPILE_OPTIONS + "$<$:--expt-relaxed-constexpr;--expt-extended-lambda;--use_fast_math>") + list(APPEND VLLM_STABLE_EXT_SRC "${KIMI_K3_ATTN_RES_SRC}") + message(STATUS + "Building Kimi K3 AttnRes for archs: ${KIMI_K3_ATTN_RES_ARCHS}") + endif() + # Hadacore kernels cuda_archs_loose_intersection(HADACORE_ARCHS "8.0+PTX;9.0+PTX" "${CUDA_ARCHS}") if(HADACORE_ARCHS) @@ -1120,6 +1138,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") target_compile_definitions(_C_stable_libtorch PRIVATE VLLM_ENABLE_COOPERATIVE_TOPK=1) endif() + if(KIMI_K3_ATTN_RES_ARCHS) + target_compile_definitions(_C_stable_libtorch PRIVATE + VLLM_ENABLE_KIMI_K3_ATTN_RES=1) + endif() # Needed by CUTLASS kernels target_compile_definitions(_C_stable_libtorch PRIVATE CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1) diff --git a/csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu b/csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu new file mode 100644 index 0000000000..eb4dcf6bf5 --- /dev/null +++ b/csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu @@ -0,0 +1,954 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + */ + +// Production AttnRes forward for Blackwell (SM100). +// +// Warp-specialized online softmax + residual + RMSNorm: +// - 1 producer warp issues cp.async.bulk row loads into shared memory. +// - 8 consumer warps compute reductions and output. +// - Q=res_weight*rms_weight remains in registers across persistent tokens. +// - V rows are converted once and cached as FP32 in TMEM between passes. +// +// Integration contract: Kimi K3 H=7168, 1<=num_blocks<=8, and token-major +// block residual storage. + +#include "../torch_utils.h" + +#include +#include +#include +#include +#include + +using bf16_t = __nv_bfloat16; + +namespace sm100 { +namespace fwd_prod_v2 { + +constexpr int K_TILE = 1024; +constexpr int N_CHUNK_DEFAULT = 4; +constexpr int CHUNK_DEPTH = 2; +constexpr int BLK = 288; // 1 producer warp + 8 consumer warps +constexpr int CONSUMER_THREADS = BLK - 32; // 256 +constexpr int CONSUMER_WARPS = CONSUMER_THREADS / 32; +constexpr int CONSUMER_GROUPS = 2; // two 128-thread consumer groups +constexpr int CONSUMER_THREADS_PER_GROUP = CONSUMER_THREADS / CONSUMER_GROUPS; +constexpr int FIRST_USER_NAMED_BARRIER = 8; + +__device__ __forceinline__ const bf16_t* residual_addr( + const bf16_t* block_res, const bf16_t* layer_res, int source, int N, + int token, int block_stride_m, int block_stride_r, int H) { + if (source < N - 1) { + return block_res + static_cast(token) * block_stride_m + + source * block_stride_r; + } + return layer_res + static_cast(token) * H; +} + +__device__ __forceinline__ uint32_t elect_one_sync() { + uint32_t pred = 0; + uint32_t laneid = 0; + asm volatile( + "{\n" + ".reg .b32 %%rx;\n" + ".reg .pred %%px;\n" + " elect.sync %%rx|%%px, %2;\n" + "@%%px mov.s32 %1, 1;\n" + " mov.s32 %0, %%rx;\n" + "}\n" + : "+r"(laneid), "+r"(pred) + : "r"(0xffffffff)); + return pred; +} + +__device__ __forceinline__ void mbarrier_init(uint64_t& barrier, + int thread_count) { + uint32_t const barrier_addr = + static_cast(__cvta_generic_to_shared(&barrier)); + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" ::"r"(barrier_addr), + "r"(thread_count)); +} + +__device__ __forceinline__ void mbarrier_expect_tx(uint64_t& barrier, + uint32_t bytes) { + uint32_t const barrier_addr = + static_cast(__cvta_generic_to_shared(&barrier)); + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" ::"r"( + barrier_addr), + "r"(bytes)); +} + +__device__ __forceinline__ void mbarrier_wait(uint64_t& barrier, int phase) { + uint32_t const barrier_addr = + static_cast(__cvta_generic_to_shared(&barrier)); + asm volatile( + "{\n" + ".reg .pred p;\n" + "WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 p, [%0], %1;\n" + "@p bra DONE;\n" + "bra WAIT;\n" + "DONE:\n" + "}\n" ::"r"(barrier_addr), + "r"(phase)); +} + +__device__ __forceinline__ void mbarrier_arrive(uint64_t& barrier) { + uint32_t const barrier_addr = + static_cast(__cvta_generic_to_shared(&barrier)); + asm volatile( + "{\n" + ".reg .b64 state;\n" + "mbarrier.arrive.shared::cta.b64 state, [%0];\n" + "}\n" ::"r"(barrier_addr)); +} + +__device__ __forceinline__ void fence_mbarrier_init() { + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} + +__device__ __forceinline__ void named_barrier_sync(uint32_t num_threads, + uint32_t user_barrier_id) { + asm volatile( + "bar.sync %0, %1;" ::"r"(user_barrier_id + FIRST_USER_NAMED_BARRIER), + "r"(num_threads) + : "memory"); +} + +__device__ __forceinline__ void tmem_allocate(int num_columns, uint32_t* dst) { + uint32_t const dst_addr = + static_cast(__cvta_generic_to_shared(dst)); + asm volatile( + "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"( + dst_addr), + "r"(num_columns)); +} + +__device__ __forceinline__ void tmem_free(uint32_t tmem_ptr, int num_columns) { + asm volatile( + "tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" ::"r"(tmem_ptr), + "r"(num_columns)); +} + +__device__ __forceinline__ void tmem_release_allocation_lock() { + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;"); +} + +__device__ __forceinline__ void tmem_store_wait() { + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); +} + +template +__device__ __forceinline__ void tmem_load(uint32_t src_addr, T* dst) { + uint32_t* values = reinterpret_cast(dst); + if constexpr (N == 8) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x8.b32" + "{%0, %1, %2, %3, %4, %5, %6, %7}, [%8];\n" + : "=r"(values[0]), "=r"(values[1]), "=r"(values[2]), "=r"(values[3]), + "=r"(values[4]), "=r"(values[5]), "=r"(values[6]), "=r"(values[7]) + : "r"(src_addr)); + } else { + static_assert(N == 4, "AttnRes TMEM helpers support x4 and x8"); + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x4.b32" + "{%0, %1, %2, %3}, [%4];\n" + : "=r"(values[0]), "=r"(values[1]), "=r"(values[2]), "=r"(values[3]) + : "r"(src_addr)); + } +} + +template +__device__ __forceinline__ void tmem_store(uint32_t dst_addr, T* src) { + uint32_t* values = reinterpret_cast(src); + if constexpr (N == 8) { + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x8.b32" + "[%8], {%0, %1, %2, %3, %4, %5, %6, %7};\n" ::"r"(values[0]), + "r"(values[1]), "r"(values[2]), "r"(values[3]), "r"(values[4]), + "r"(values[5]), "r"(values[6]), "r"(values[7]), "r"(dst_addr)); + } else { + static_assert(N == 4, "AttnRes TMEM helpers support x4 and x8"); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x4.b32" + "[%4], {%0, %1, %2, %3};\n" ::"r"(values[0]), + "r"(values[1]), "r"(values[2]), "r"(values[3]), "r"(dst_addr)); + } +} + +__device__ __forceinline__ float2 float2_add(const float2& a, const float2& b) { + float2 result; + asm volatile("add.rn.f32x2 %0, %1, %2;\n" + : "=l"(reinterpret_cast(result)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b))); + return result; +} + +__device__ __forceinline__ float2 float2_mul(const float2& a, const float2& b) { + float2 result; + asm volatile("mul.f32x2 %0, %1, %2;\n" + : "=l"(reinterpret_cast(result)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b))); + return result; +} + +__device__ __forceinline__ float2 float2_fma(const float2& a, const float2& b, + const float2& c) { + float2 result; + asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n" + : "=l"(reinterpret_cast(result)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b)), + "l"(reinterpret_cast(c))); + return result; +} + +template +struct FwdSmemPlan { + alignas(16) uint64_t bar_ready[CHUNK_DEPTH]; + alignas(16) uint64_t bar_consumed[CHUNK_DEPTH]; + alignas(16) uint64_t bar_output_norm_ready; + alignas(16) float2 ws_stats[CONSUMER_WARPS][NC]; + uint32_t tmem_base; +}; + +__device__ __forceinline__ void cp_async_bulk(void* smem_dst, + const void* gmem_src, int bytes, + uint64_t& mbar) { + uint32_t const s = static_cast(__cvta_generic_to_shared(smem_dst)); + uint32_t const m = static_cast(__cvta_generic_to_shared(&mbar)); + asm volatile( + "cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [%0], " + "[%1], %2, [%3];\n" ::"r"(s), + "l"(gmem_src), "r"(bytes), "r"(m) + : "memory"); +} + +template +__global__ void __launch_bounds__(BLK, 1) attn_res_fwd_online_v2_kernel( + const bf16_t* __restrict__ block_res, bf16_t* __restrict__ layer_res, + const bf16_t* __restrict__ delta, const bf16_t* __restrict__ res_w, + const bf16_t* __restrict__ rms_w, bf16_t* __restrict__ output, int N, int T, + int B, int block_stride_m, int block_stride_r, float rms_eps, + const bf16_t* __restrict__ output_norm_weight, float output_norm_eps) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && __CUDA_ARCH__ < 1100 + constexpr float LOG2_E = 1.4426950408889634f; + constexpr int N_CHUNK = NC; + // The two-source specialization only consumes half of the TMEM columns. + constexpr int TMEM_COLS_ALLOC = NC == 2 ? 128 : 256; + constexpr int NUM_BUFS = CHUNK_DEPTH * NC; + constexpr int NHT = H / K_TILE; + constexpr int SLICES_PER_GROUP = + (NHT + CONSUMER_GROUPS - 1) / CONSUMER_GROUPS; + constexpr int VEC = 8; + constexpr int ACC_PER_THREAD = H == 7168 ? 28 : SLICES_PER_GROUP * VEC; + constexpr int TMEM_V_COLS_PER_GROUP = SLICES_PER_GROUP * N_CHUNK * VEC; + constexpr int TMEM_V_COLS_TOTAL = CONSUMER_GROUPS * TMEM_V_COLS_PER_GROUP; + static_assert(TMEM_V_COLS_TOTAL <= TMEM_COLS_ALLOC); + static_assert(H >= 4096 && H <= 8192); + static_assert(H % K_TILE == 0); + + const int tid = threadIdx.x; + const int wid = tid >> 5; + const int lane = tid & 31; + const int TB = T * B; + const int num_ctas = gridDim.x; + const int num_chunks = (N + N_CHUNK - 1) / N_CHUNK; + + const int comp_wid = wid - 1; + const int comp_tid = tid - 32; + const int group = (comp_wid >= 4) ? 1 : 0; + const int ct_in_group = + (comp_tid >= 0) ? (comp_tid & (CONSUMER_THREADS_PER_GROUP - 1)) : -1; + const int k_local = ct_in_group * VEC; + + constexpr size_t V_BYTES = (size_t)NUM_BUFS * H * sizeof(bf16_t); + constexpr size_t DELTA_BYTES = + HAS_DELTA ? (size_t)CHUNK_DEPTH * H * sizeof(bf16_t) : 0; + constexpr size_t OUTPUT_NORM_BYTES = + OUTPUT_NORM_IN_SMEM ? (size_t)H * sizeof(bf16_t) : 0; + extern __shared__ __align__(16) char smem_raw[]; + bf16_t* v_bufs = reinterpret_cast(smem_raw); // [NUM_BUFS][H] + bf16_t* delta_bufs = reinterpret_cast(smem_raw + V_BYTES); + bf16_t* output_norm_buf = + reinterpret_cast(smem_raw + V_BYTES + DELTA_BYTES); + FwdSmemPlan& plan = *reinterpret_cast*>( + smem_raw + V_BYTES + DELTA_BYTES + OUTPUT_NORM_BYTES); + + auto slot_of = [](long long gci, int n) { + return (int)(gci % CHUNK_DEPTH) * N_CHUNK + n; + }; + auto phase_of = [](long long gci) { return (int)((gci / CHUNK_DEPTH) & 1); }; + auto buf_ptr = [&](int slot) -> bf16_t* { return v_bufs + slot * H; }; + auto delta_buf_ptr = [&](int chunk_slot) -> bf16_t* { + return delta_bufs + chunk_slot * H; + }; + + if (wid == 0 && elect_one_sync()) { + #pragma unroll + for (int i = 0; i < CHUNK_DEPTH; i++) { + mbarrier_init(plan.bar_ready[i], 1); + mbarrier_init(plan.bar_consumed[i], CONSUMER_WARPS); + } + if constexpr (OUTPUT_NORM_IN_SMEM) { + mbarrier_init(plan.bar_output_norm_ready, 1); + } + fence_mbarrier_init(); + } + + // gdc wait BEFORE tmem alloc + cudaGridDependencySynchronize(); + + if (wid == 1) { + tmem_allocate(TMEM_COLS_ALLOC, &plan.tmem_base); + if constexpr (RELEASE_TMEM) { + tmem_release_allocation_lock(); + } + } + __syncthreads(); + + if constexpr (OUTPUT_NORM_IN_SMEM) { + if (wid == 0 && elect_one_sync()) { + mbarrier_expect_tx(plan.bar_output_norm_ready, H * (int)sizeof(bf16_t)); + cp_async_bulk(output_norm_buf, output_norm_weight, H * sizeof(bf16_t), + plan.bar_output_norm_ready); + } + } + + const uint32_t my_v_tmem = + comp_tid >= 0 ? plan.tmem_base + group * TMEM_V_COLS_PER_GROUP : 0; + float q_cache[ACC_PER_THREAD]; + if (comp_tid >= 0) { + #pragma unroll + for (int si = 0; si < SLICES_PER_GROUP; si++) { + if constexpr (H == 7168) { + if (si == SLICES_PER_GROUP - 1) { + int h_base = 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4; + #pragma unroll + for (int j = 0; j < 4; j++) { + int h = h_base + j; + q_cache[si * VEC + j] = + __bfloat162float(rms_w[h]) * __bfloat162float(res_w[h]); + } + continue; + } + } + int dt = si * CONSUMER_GROUPS + group; + if (dt >= NHT) continue; + int h_base = dt * K_TILE + k_local; + #pragma unroll + for (int j = 0; j < VEC; j++) { + int h = h_base + j; + q_cache[si * VEC + j] = + __bfloat162float(rms_w[h]) * __bfloat162float(res_w[h]); + } + } + } + + if (wid == 0) { + if (elect_one_sync()) { + long long gci = 0; + for (int tb = blockIdx.x; tb < TB; tb += num_ctas) { + const int t = tb / B; + for (int ci = 0; ci < num_chunks; ci++, gci++) { + int ns = ci * N_CHUNK; + int an = min(N_CHUNK, N - ns); + int chunk_slot = (int)(gci % CHUNK_DEPTH); + int pc = phase_of(gci); + mbarrier_wait(plan.bar_consumed[chunk_slot], pc ^ 1); + int transaction_bytes = an * H * (int)sizeof(bf16_t); + if constexpr (HAS_DELTA) { + int prefix_n = N - 1 - ns; + if (prefix_n >= 0 && prefix_n < an) { + transaction_bytes += H * (int)sizeof(bf16_t); + } + } + mbarrier_expect_tx(plan.bar_ready[chunk_slot], transaction_bytes); + #pragma unroll + for (int n = 0; n < N_CHUNK; n++) { + if (n >= an) continue; + int slot = slot_of(gci, n); + const bf16_t* src = + residual_addr(block_res, layer_res, ns + n, N, t, + block_stride_m, block_stride_r, H); + cp_async_bulk(buf_ptr(slot), src, H * sizeof(bf16_t), + plan.bar_ready[chunk_slot]); + } + if constexpr (HAS_DELTA) { + int prefix_n = N - 1 - ns; + if (prefix_n >= 0 && prefix_n < an) { + cp_async_bulk(delta_buf_ptr(chunk_slot), + delta + (long long)tb * H, H * sizeof(bf16_t), + plan.bar_ready[chunk_slot]); + } + } + } + } + } + } else { + float acc32[ACC_PER_THREAD] = {}; + float eps_cache; + asm volatile("mov.b32 %0, %1;" : "=f"(eps_cache) : "f"(rms_eps)); + + long long gci = 0; + for (int tb = blockIdx.x; tb < TB; tb += num_ctas) { + float m_running = -FLT_MAX; + float s_running = 0.f; + #pragma unroll + for (int i = 0; i < ACC_PER_THREAD; i++) { + acc32[i] = 0.f; + } + + for (int ci = 0; ci < num_chunks; ci++, gci++) { + int ns = ci * N_CHUNK; + int an = min(N_CHUNK, N - ns); + int chunk_slot = (int)(gci % CHUNK_DEPTH); + int pr = phase_of(gci); + mbarrier_wait(plan.bar_ready[chunk_slot], pr); + + float2 sq_local[N_CHUNK] = {}; + float2 dot_local[N_CHUNK] = {}; + + auto pass_A_body = [&](auto AN_TOK) { + constexpr int AN = decltype(AN_TOK)::value; + #pragma unroll + for (int si = 0; si < SLICES_PER_GROUP; si++) { + if constexpr (H == 7168) { + if (si == SLICES_PER_GROUP - 1) { + int h_base = + 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4; + const float* qv = &q_cache[si * VEC]; + #pragma unroll + for (int n = 0; n < AN; n++) { + int slot = slot_of(gci, n); + int2 vp = + *reinterpret_cast(buf_ptr(slot) + h_base); + auto* v2 = reinterpret_cast<__nv_bfloat162*>(&vp); + if constexpr (HAS_DELTA) { + int prefix_n = N - 1 - ns; + if (n == prefix_n) { + const bf16_t* delta_ptr = + delta_buf_ptr(chunk_slot) + h_base; + #pragma unroll + for (int j = 0; j < 2; j++) { + auto delta2 = *reinterpret_cast( + delta_ptr + 2 * j); + v2[j] = __hadd2(v2[j], delta2); + } + *reinterpret_cast(layer_res + (long long)tb * H + + h_base) = vp; + } + } + float2 f[2] = {__bfloat1622float2(v2[0]), + __bfloat1622float2(v2[1])}; + tmem_store<4>(my_v_tmem + (si * N_CHUNK + n) * VEC, f); + sq_local[n] = float2_fma(f[0], f[0], sq_local[n]); + sq_local[n] = float2_fma(f[1], f[1], sq_local[n]); + dot_local[n] = + float2_fma(f[0], make_float2(qv[0], qv[1]), dot_local[n]); + dot_local[n] = + float2_fma(f[1], make_float2(qv[2], qv[3]), dot_local[n]); + } + continue; + } + } + int dt = si * CONSUMER_GROUPS + group; + if (dt >= NHT) continue; + int h_base = dt * K_TILE + k_local; + const float* qv = &q_cache[si * VEC]; + + #pragma unroll + for (int n = 0; n < AN; n++) { + int slot = slot_of(gci, n); + int4 vp = *reinterpret_cast(buf_ptr(slot) + h_base); + auto* v2 = reinterpret_cast<__nv_bfloat162*>(&vp); + if constexpr (HAS_DELTA) { + int prefix_n = N - 1 - ns; + if (n == prefix_n) { + const bf16_t* delta_ptr = delta_buf_ptr(chunk_slot) + h_base; + #pragma unroll + for (int j = 0; j < VEC / 2; j++) { + auto delta2 = *reinterpret_cast( + delta_ptr + 2 * j); + v2[j] = __hadd2(v2[j], delta2); + } + *reinterpret_cast(layer_res + (long long)tb * H + + h_base) = vp; + } + } + float2 f[4] = { + __bfloat1622float2(v2[0]), __bfloat1622float2(v2[1]), + __bfloat1622float2(v2[2]), __bfloat1622float2(v2[3])}; + tmem_store(my_v_tmem + (si * N_CHUNK + n) * VEC, f); + #pragma unroll + for (int j = 0; j < VEC / 2; j++) { + sq_local[n] = float2_fma(f[j], f[j], sq_local[n]); + dot_local[n] = float2_fma( + f[j], make_float2(qv[2 * j], qv[2 * j + 1]), dot_local[n]); + } + } + } + }; + if constexpr (NC == 4) { + switch (an) { + case 4: + pass_A_body(std::integral_constant{}); + break; + case 3: + pass_A_body(std::integral_constant{}); + break; + case 2: + pass_A_body(std::integral_constant{}); + break; + case 1: + pass_A_body(std::integral_constant{}); + break; + default: + __builtin_unreachable(); + } + } else if constexpr (NC == 3) { + switch (an) { + case 3: + pass_A_body(std::integral_constant{}); + break; + case 2: + pass_A_body(std::integral_constant{}); + break; + case 1: + pass_A_body(std::integral_constant{}); + break; + default: + __builtin_unreachable(); + } + } else { + static_assert(NC == 2); + switch (an) { + case 2: + pass_A_body(std::integral_constant{}); + break; + case 1: + pass_A_body(std::integral_constant{}); + break; + default: + __builtin_unreachable(); + } + } + if (lane == 0) { + mbarrier_arrive(plan.bar_consumed[chunk_slot]); + } + tmem_store_wait(); + + float2 reduce_pair[N_CHUNK]; + #pragma unroll + for (int n = 0; n < N_CHUNK; n++) { + reduce_pair[n] = make_float2(sq_local[n].x + sq_local[n].y, + dot_local[n].x + dot_local[n].y); + } + #pragma unroll + for (int offset = 16; offset > 0; offset >>= 1) { + #pragma unroll + for (int n = 0; n < N_CHUNK; n++) { + uint64_t packed = reinterpret_cast(reduce_pair[n]); + packed = __shfl_xor_sync(0xffffffff, packed, offset); + float2 other = reinterpret_cast(packed); + reduce_pair[n] = float2_add(reduce_pair[n], other); + } + } + if (lane == 0) { + #pragma unroll + for (int n = 0; n < N_CHUNK; n++) { + plan.ws_stats[comp_wid][n] = reduce_pair[n]; + } + } + named_barrier_sync(CONSUMER_THREADS, 0); + + float local_rsig = 0.f; + float local_logit = 0.f; + int stat_n = lane / CONSUMER_WARPS; + int stat_w = lane % CONSUMER_WARPS; + float2 totals = {}; + if (stat_n < N_CHUNK) { + totals = plan.ws_stats[stat_w][stat_n]; + } + #pragma unroll + for (int offset = CONSUMER_WARPS / 2; offset > 0; offset >>= 1) { + totals.x += + __shfl_down_sync(0xffffffff, totals.x, offset, CONSUMER_WARPS); + totals.y += + __shfl_down_sync(0xffffffff, totals.y, offset, CONSUMER_WARPS); + } + if (stat_n < N_CHUNK && stat_w == 0) { + local_rsig = rsqrtf(totals.x / H + eps_cache); + local_logit = totals.y * local_rsig; + } + float logit_n[N_CHUNK]; + #pragma unroll + for (int n = 0; n < N_CHUNK; n++) { + logit_n[n] = __shfl_sync(0xffffffff, local_logit, n * CONSUMER_WARPS); + } + + float m_chunk = -FLT_MAX; + #pragma unroll + for (int n = 0; n < N_CHUNK; n++) { + if (n < an) m_chunk = fmaxf(m_chunk, logit_n[n]); + } + float m_new = fmaxf(m_running, m_chunk); + float corr = exp2f((m_running - m_new) * LOG2_E); + float w_n[N_CHUNK] = {}; + float w_sum = 0.f; + #pragma unroll + for (int n = 0; n < N_CHUNK; n++) { + if (n < an) { + w_n[n] = exp2f((logit_n[n] - m_new) * LOG2_E); + w_sum += w_n[n]; + } + } + + auto pass_B_body = [&](auto AN_TOK) { + constexpr int AN = decltype(AN_TOK)::value; + #pragma unroll + for (int si = 0; si < SLICES_PER_GROUP; si++) { + if constexpr (H == 7168) { + if (si == SLICES_PER_GROUP - 1) { + float2 corr2 = make_float2(corr, corr); + float2 a[2]; + #pragma unroll + for (int j = 0; j < 2; j++) { + float2 old = make_float2(acc32[si * VEC + 2 * j], + acc32[si * VEC + 2 * j + 1]); + a[j] = float2_mul(old, corr2); + } + float2 f_cache[AN][2]; + #pragma unroll + for (int n = 0; n < AN; n++) { + tmem_load<4>(my_v_tmem + (si * N_CHUNK + n) * VEC, + f_cache[n]); + } + #pragma unroll + for (int n = 0; n < AN; n++) { + float2 wn = make_float2(w_n[n], w_n[n]); + #pragma unroll + for (int j = 0; j < 2; j++) { + a[j] = float2_fma(wn, f_cache[n][j], a[j]); + } + } + #pragma unroll + for (int j = 0; j < 2; j++) { + acc32[si * VEC + 2 * j] = a[j].x; + acc32[si * VEC + 2 * j + 1] = a[j].y; + } + continue; + } + } + int dt = si * CONSUMER_GROUPS + group; + if (dt >= NHT) continue; + float2 corr2 = make_float2(corr, corr); + float2 a[VEC / 2]; + #pragma unroll + for (int j = 0; j < VEC / 2; j++) { + float2 old = make_float2(acc32[si * VEC + 2 * j], + acc32[si * VEC + 2 * j + 1]); + a[j] = float2_mul(old, corr2); + } + float2 f_cache[AN][VEC / 2]; + #pragma unroll + for (int n = 0; n < AN; n++) { + tmem_load(my_v_tmem + (si * N_CHUNK + n) * VEC, f_cache[n]); + } + #pragma unroll + for (int n = 0; n < AN; n++) { + float2 wn = make_float2(w_n[n], w_n[n]); + #pragma unroll + for (int j = 0; j < VEC / 2; j++) { + a[j] = float2_fma(wn, f_cache[n][j], a[j]); + } + } + #pragma unroll + for (int j = 0; j < VEC / 2; j++) { + acc32[si * VEC + 2 * j] = a[j].x; + acc32[si * VEC + 2 * j + 1] = a[j].y; + } + } + }; + if constexpr (NC == 4) { + switch (an) { + case 4: + pass_B_body(std::integral_constant{}); + break; + case 3: + pass_B_body(std::integral_constant{}); + break; + case 2: + pass_B_body(std::integral_constant{}); + break; + case 1: + pass_B_body(std::integral_constant{}); + break; + default: + __builtin_unreachable(); + } + } else if constexpr (NC == 3) { + switch (an) { + case 3: + pass_B_body(std::integral_constant{}); + break; + case 2: + pass_B_body(std::integral_constant{}); + break; + case 1: + pass_B_body(std::integral_constant{}); + break; + default: + __builtin_unreachable(); + } + } else { + static_assert(NC == 2); + switch (an) { + case 2: + pass_B_body(std::integral_constant{}); + break; + case 1: + pass_B_body(std::integral_constant{}); + break; + default: + __builtin_unreachable(); + } + } + + s_running = s_running * corr + w_sum; + m_running = m_new; + } + + float inv_s = 1.f / s_running; + bf16_t* out_ptr = output + (long long)tb * H; + float2 output_sq_pair = {}; + // When output RMSNorm is fused, the softmax denominator cancels: + // (acc / s) * rsqrt(mean((acc / s)^2) + eps) + // = acc * rsqrt(mean(acc^2) + eps * s^2). + #pragma unroll + for (int si = 0; si < SLICES_PER_GROUP; si++) { + if constexpr (H == 7168) { + if (si == SLICES_PER_GROUP - 1) { + int h_base = 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4; + uint2 packed; + auto* ov2 = reinterpret_cast<__nv_bfloat162*>(&packed); + float2 inv2 = make_float2(inv_s, inv_s); + #pragma unroll + for (int j = 0; j < 2; j++) { + float2 old = make_float2(acc32[si * VEC + 2 * j], + acc32[si * VEC + 2 * j + 1]); + if constexpr (HAS_OUTPUT_NORM) { + output_sq_pair = float2_fma(old, old, output_sq_pair); + } else { + float2 mixed = float2_mul(old, inv2); + ov2[j] = __float22bfloat162_rn(mixed); + } + } + if constexpr (!HAS_OUTPUT_NORM) { + *reinterpret_cast(out_ptr + h_base) = packed; + } + continue; + } + } + int dt = si * CONSUMER_GROUPS + group; + if (dt >= NHT) continue; + int h_base = dt * K_TILE + k_local; + uint4 packed; + auto* ov2 = reinterpret_cast<__nv_bfloat162*>(&packed); + float2 inv2 = make_float2(inv_s, inv_s); + #pragma unroll + for (int j = 0; j < VEC / 2; j++) { + float2 old = + make_float2(acc32[si * VEC + 2 * j], acc32[si * VEC + 2 * j + 1]); + if constexpr (HAS_OUTPUT_NORM) { + output_sq_pair = float2_fma(old, old, output_sq_pair); + } else { + float2 mixed = float2_mul(old, inv2); + ov2[j] = __float22bfloat162_rn(mixed); + } + } + if constexpr (!HAS_OUTPUT_NORM) { + *reinterpret_cast(out_ptr + h_base) = packed; + } + } + + if constexpr (HAS_OUTPUT_NORM) { + if constexpr (OUTPUT_NORM_IN_SMEM) { + // The immutable weight copy is acquired once, at its first use. + if (tb == blockIdx.x) { + mbarrier_wait(plan.bar_output_norm_ready, 0); + } + } + float output_sq = output_sq_pair.x + output_sq_pair.y; + #pragma unroll + for (int offset = 16; offset > 0; offset >>= 1) { + output_sq += __shfl_xor_sync(0xffffffff, output_sq, offset); + } + if (lane == 0) { + plan.ws_stats[comp_wid][0] = make_float2(output_sq, 0.f); + } + named_barrier_sync(CONSUMER_THREADS, 0); + float total_sq = lane < CONSUMER_WARPS ? plan.ws_stats[lane][0].x : 0.f; + #pragma unroll + for (int offset = CONSUMER_WARPS / 2; offset > 0; offset >>= 1) { + total_sq += + __shfl_down_sync(0xffffffff, total_sq, offset, CONSUMER_WARPS); + } + if (lane == 0) { + total_sq = + rsqrtf(total_sq / H + output_norm_eps * s_running * s_running); + } + float output_rsigma = __shfl_sync(0xffffffff, total_sq, 0); + #pragma unroll + for (int si = 0; si < SLICES_PER_GROUP; si++) { + if constexpr (H == 7168) { + if (si == SLICES_PER_GROUP - 1) { + int h_base = 6 * K_TILE + group * (K_TILE / 2) + ct_in_group * 4; + uint2 packed; + auto* values = reinterpret_cast(&packed); + #pragma unroll + for (int j = 0; j < 4; j++) { + const bf16_t* weight_ptr = + OUTPUT_NORM_IN_SMEM ? output_norm_buf : output_norm_weight; + float weight = __bfloat162float(weight_ptr[h_base + j]); + values[j] = __float2bfloat16(acc32[si * VEC + j] * + output_rsigma * weight); + } + *reinterpret_cast(out_ptr + h_base) = packed; + continue; + } + } + int dt = si * CONSUMER_GROUPS + group; + if (dt >= NHT) continue; + int h_base = dt * K_TILE + k_local; + uint4 packed; + auto* values = reinterpret_cast(&packed); + #pragma unroll + for (int j = 0; j < VEC; j++) { + const bf16_t* weight_ptr = + OUTPUT_NORM_IN_SMEM ? output_norm_buf : output_norm_weight; + float weight = __bfloat162float(weight_ptr[h_base + j]); + values[j] = + __float2bfloat16(acc32[si * VEC + j] * output_rsigma * weight); + } + *reinterpret_cast(out_ptr + h_base) = packed; + } + } + } + } + + cudaTriggerProgrammaticLaunchCompletion(); + __syncthreads(); + if (wid == 1) { + tmem_free(plan.tmem_base, TMEM_COLS_ALLOC); + } +#else + if (threadIdx.x == 0) { + printf("attn_res_fwd_online_v2_kernel requires sm_10x\n"); + } +#endif +} + +template +static void launch_fwd(const bf16_t* block_residual, bf16_t* layer_residual, + const bf16_t* delta, const bf16_t* res_weight, + const bf16_t* rms_weight, bf16_t* output, int N, int T, + int B, float rms_eps, int num_sm, cudaStream_t stream, + const bf16_t* output_norm_weight = nullptr, + float output_norm_eps = 0.f, int block_stride_m = 0, + int block_stride_r = 0) { + constexpr size_t smem_size = + ((size_t)CHUNK_DEPTH * (NC + (HAS_DELTA ? 1 : 0)) * H * sizeof(bf16_t) + + (OUTPUT_NORM_IN_SMEM ? (size_t)H * sizeof(bf16_t) : 0) + + sizeof(FwdSmemPlan) + 15) & + ~size_t(15); + auto kernel = + &attn_res_fwd_online_v2_kernel; + static bool attrs_set = false; + if (!attrs_set) { + if (smem_size > 48 * 1024) { + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + smem_size); + } + attrs_set = true; + } + int grid = RELEASE_TMEM ? num_sm * 2 : num_sm; + cudaLaunchConfig_t config{}; + config.gridDim = grid; + config.blockDim = BLK; + config.dynamicSmemBytes = smem_size; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = 1; + config.attrs = attrs; + config.numAttrs = 1; + cudaLaunchKernelEx(&config, kernel, block_residual, layer_residual, delta, + res_weight, rms_weight, output, N, T, B, block_stride_m, + block_stride_r, rms_eps, output_norm_weight, + output_norm_eps); +} + +} // namespace fwd_prod_v2 +} // namespace sm100 + +void kimi_k3_attn_res(torch::stable::Tensor& prefix, + torch::stable::Tensor const& delta, + torch::stable::Tensor const& blocks, + torch::stable::Tensor const& norm_weight, + torch::stable::Tensor const& qk_weight, + torch::stable::Tensor const& output_norm_weight, + torch::stable::Tensor& output, int64_t num_blocks, + double eps, double output_norm_eps) { + int const num_tokens = static_cast(prefix.size(0)); + int const device = prefix.get_device_index(); + torch::stable::accelerator::DeviceGuard const device_guard(device); + cudaDeviceProp const* properties = get_device_prop(); + STD_TORCH_CHECK(properties->major == 10, + "Kimi K3 AttnRes requires the SM100 family"); + + using namespace sm100::fwd_prod_v2; + // Two-source chunks and two resident CTAs are beneficial once setup is + // amortized by the long, full eight-block prefill workload. + if (num_blocks == 8 && num_tokens >= 4096) { + launch_fwd<7168, 2, true, true, true, true>( + static_cast(blocks.data_ptr()), + static_cast(prefix.data_ptr()), + static_cast(delta.data_ptr()), + static_cast(qk_weight.data_ptr()), + static_cast(norm_weight.data_ptr()), + static_cast(output.data_ptr()), + static_cast(num_blocks) + 1, num_tokens, 1, + static_cast(eps), properties->multiProcessorCount, + get_current_cuda_stream(device), + static_cast(output_norm_weight.data_ptr()), + static_cast(output_norm_eps), static_cast(blocks.stride(0)), + static_cast(blocks.stride(1))); + } else { + launch_fwd<7168, 4, false, true, true, true>( + static_cast(blocks.data_ptr()), + static_cast(prefix.data_ptr()), + static_cast(delta.data_ptr()), + static_cast(qk_weight.data_ptr()), + static_cast(norm_weight.data_ptr()), + static_cast(output.data_ptr()), + static_cast(num_blocks) + 1, num_tokens, 1, + static_cast(eps), properties->multiProcessorCount, + get_current_cuda_stream(device), + static_cast(output_norm_weight.data_ptr()), + static_cast(output_norm_eps), static_cast(blocks.stride(0)), + static_cast(blocks.stride(1))); + } + cudaError_t const error = cudaGetLastError(); + STD_TORCH_CHECK( + error == cudaSuccess, + "Kimi K3 AttnRes kernel launch failed: ", cudaGetErrorString(error)); +} diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 3834bea585..5a9c91d456 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -315,6 +315,17 @@ void fused_minimax_m3_qknorm_rope_kv_insert( std::optional index_q_out, const std::string& kv_cache_dtype, bool skip_index_branch); +#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES +void kimi_k3_attn_res(torch::stable::Tensor& prefix, + torch::stable::Tensor const& delta, + torch::stable::Tensor const& blocks, + torch::stable::Tensor const& norm_weight, + torch::stable::Tensor const& qk_weight, + torch::stable::Tensor const& output_norm_weight, + torch::stable::Tensor& output, int64_t num_blocks, + double eps, double output_norm_eps); +#endif + // Sampler kernels (shared CUDA/ROCm) void apply_repetition_penalties_( torch::stable::Tensor& logits, const torch::stable::Tensor& prompt_mask, diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 0a475d02c6..c364e21147 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -468,6 +468,14 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "int block_size, Tensor!? q_out, Tensor!? index_q_out, " "str kv_cache_dtype, bool skip_index_branch=False) -> ()"); +#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES + ops.def( + "kimi_k3_attn_res(" + "Tensor! prefix, Tensor delta, Tensor blocks, Tensor norm_weight, " + "Tensor qk_weight, Tensor output_norm_weight, Tensor! output, " + "int num_blocks, float eps, float output_norm_eps) -> ()"); +#endif + // Apply repetition penalties to logits in-place. ops.def( "apply_repetition_penalties_(Tensor! logits, Tensor prompt_mask, " @@ -693,6 +701,9 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) { #endif ops.impl("fused_minimax_m3_qknorm_rope_kv_insert", TORCH_BOX(&fused_minimax_m3_qknorm_rope_kv_insert)); +#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES + ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res)); +#endif // Sampler kernels (shared CUDA/ROCm) ops.impl("apply_repetition_penalties_", diff --git a/tests/models/kimi_k3/test_amd_attn_res.py b/tests/models/kimi_k3/test_amd_attn_res.py new file mode 100644 index 0000000000..f6dfaa422b --- /dev/null +++ b/tests/models/kimi_k3/test_amd_attn_res.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +import torch.nn.functional as F + +from vllm.models.kimi_k3.amd.ops.attn_res import attn_res +from vllm.platforms import current_platform + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), + reason="AMD AttnRes requires ROCm", +) + + +def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor: + storage = torch.randn( + *shape[:-1], + shape[-1] + padding, + device="cuda", + dtype=torch.bfloat16, + ) + return storage[..., : shape[-1]] + + +def _reference( + prefix: torch.Tensor, + blocks: torch.Tensor, + norm_weight: torch.Tensor, + qk_weight: torch.Tensor, + num_blocks: int, + eps: float, +) -> torch.Tensor: + hidden_size = prefix.shape[-1] + values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1) + keys = F.rms_norm(values, (hidden_size,), norm_weight, eps) + probs = (keys @ qk_weight).softmax(dim=-1) + return torch.matmul(probs.unsqueeze(1), values).squeeze(1) + + +@pytest.mark.parametrize( + ( + "num_tokens", + "num_blocks", + "block_capacity", + "hidden_size", + "row_padding", + ), + [ + pytest.param(0, 3, 5, 128, 0, id="empty"), + pytest.param(1, 1, 2, 128, 0, id="decode-single"), + pytest.param(17, 4, 6, 1024, 7, id="decode-padded"), + pytest.param(320, 8, 10, 7168, 0, id="prefill-full"), + ], +) +def test_amd_attn_res_matches_reference( + num_tokens: int, + num_blocks: int, + block_capacity: int, + hidden_size: int, + row_padding: int, +) -> None: + eps = 1e-5 + prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=row_padding) + blocks = _randn_with_row_padding( + num_tokens, + block_capacity, + hidden_size, + padding=row_padding, + ) + norm_weight = 1 + 0.1 * torch.randn( + hidden_size, device="cuda", dtype=torch.bfloat16 + ) + qk_weight = ( + torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5 + ) + expected = _reference( + prefix, + blocks, + norm_weight, + qk_weight, + num_blocks, + eps, + ) + original_prefix = prefix.clone() + original_blocks = blocks.clone() + + actual = attn_res( + prefix, + blocks, + norm_weight, + qk_weight, + num_blocks, + eps, + ) + + torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2) + torch.testing.assert_close(prefix, original_prefix, atol=0, rtol=0) + torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0) + assert actual.shape == prefix.shape + assert actual.is_contiguous() diff --git a/tests/models/kimi_k3/test_attn_res.py b/tests/models/kimi_k3/test_attn_res.py new file mode 100644 index 0000000000..69c9213647 --- /dev/null +++ b/tests/models/kimi_k3/test_attn_res.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +import torch.nn.functional as F + +from vllm.models.kimi_k3.nvidia.ops import attn_res +from vllm.platforms import current_platform + +HIDDEN_SIZE = 7168 +MAX_BLOCKS = 8 +EPS = 1e-5 + + +def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor: + storage = torch.randn( + *shape[:-1], + shape[-1] + padding, + device="cuda", + dtype=torch.bfloat16, + ) + return storage[..., : shape[-1]] + + +def _reference( + prefix: torch.Tensor, + delta: torch.Tensor | None, + blocks: torch.Tensor, + norm_weight: torch.Tensor, + qk_weight: torch.Tensor, + output_norm_weight: torch.Tensor | None, + num_blocks: int, +) -> tuple[torch.Tensor, torch.Tensor]: + if delta is not None: + prefix = prefix + delta + values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1) + keys = F.rms_norm(values, (HIDDEN_SIZE,), norm_weight, EPS) + probs = (keys @ qk_weight).softmax(dim=-1) + output = torch.matmul(probs.unsqueeze(1), values).squeeze(1) + if output_norm_weight is not None: + output = F.rms_norm(output, (HIDDEN_SIZE,), output_norm_weight, EPS) + return output, prefix + + +@pytest.mark.parametrize( + ( + "num_tokens", + "num_blocks", + "row_padding", + "write_block", + "has_delta", + "backend", + ), + [ + pytest.param(1, 0, 0, True, False, "triton", id="triton-empty"), + pytest.param(1, 0, 0, True, True, "triton", id="triton-empty-add"), + pytest.param(17, 5, 7, True, False, "triton", id="triton-write"), + pytest.param(17, 5, 7, True, True, "triton", id="triton-write-add"), + pytest.param(3, 8, 0, False, False, "triton", id="triton-full"), + pytest.param(3, 8, 0, False, True, "triton", id="triton-full-add"), + pytest.param(320, 1, 0, False, True, "nvidia", id="nvidia-1"), + pytest.param(320, 4, 0, False, True, "nvidia", id="nvidia-4"), + pytest.param(320, 8, 0, False, True, "nvidia", id="nvidia-8"), + ], +) +def test_attn_res( + num_tokens: int, + num_blocks: int, + row_padding: int, + write_block: bool, + has_delta: bool, + backend: str, +): + if backend == "nvidia" and not current_platform.is_device_capability_family(100): + pytest.skip("NVIDIA AttnRes requires the SM100 family") + + prefix = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding) + delta = ( + _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding) + if has_delta + else None + ) + blocks = _randn_with_row_padding( + num_tokens, MAX_BLOCKS, HIDDEN_SIZE, padding=row_padding + ) + norm_weight = 1 + 0.1 * torch.randn( + HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16 + ) + qk_weight = ( + torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5 + ) + output_norm_weight = 1 + 0.1 * torch.randn( + HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16 + ) + original_blocks = blocks.clone() + expected, expected_prefix = _reference( + prefix.clone(), + delta, + blocks, + norm_weight, + qk_weight, + output_norm_weight, + num_blocks, + ) + block_write_idx = num_blocks if write_block else -1 + + actual = attn_res( + prefix, + delta, + blocks, + norm_weight, + qk_weight, + output_norm_weight, + num_blocks, + block_write_idx, + EPS, + EPS, + ) + + torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2) + torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0) + if write_block: + original_blocks[:, block_write_idx].copy_(expected_prefix) + torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0) + assert actual.is_contiguous() + + +@pytest.mark.parametrize("num_blocks", range(MAX_BLOCKS + 1)) +def test_attn_res_block_counts(num_blocks: int): + prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) + blocks = torch.randn( + 1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16 + ) + norm_weight = torch.ones(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) + qk_weight = ( + torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5 + ) + output_norm_weight = torch.ones_like(norm_weight) + expected, _ = _reference( + prefix.clone(), + None, + blocks, + norm_weight, + qk_weight, + output_norm_weight, + num_blocks, + ) + + actual = attn_res( + prefix, + None, + blocks, + norm_weight, + qk_weight, + output_norm_weight, + num_blocks, + -1, + EPS, + EPS, + ) + + torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2) + + +def test_attn_res_without_output_norm(): + prefix = torch.randn(7, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) + delta = torch.randn_like(prefix) + blocks = torch.randn( + 7, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16 + ) + norm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) + qk_weight = ( + torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5 + ) + expected, _ = _reference( + prefix.clone(), delta, blocks, norm_weight, qk_weight, None, MAX_BLOCKS + ) + + actual = attn_res( + prefix, + delta, + blocks, + norm_weight, + qk_weight, + None, + MAX_BLOCKS, + -1, + EPS, + 0.0, + ) + + torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 07a3a583f9..b1d2f1344d 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2746,6 +2746,33 @@ def concat_and_cache_mla( ) +def kimi_k3_attn_res( + prefix: torch.Tensor, + delta: torch.Tensor, + blocks: torch.Tensor, + norm_weight: torch.Tensor, + qk_weight: torch.Tensor, + output_norm_weight: torch.Tensor, + num_blocks: int, + eps: float, + output_norm_eps: float, +) -> torch.Tensor: + output = torch.empty_like(prefix) + torch.ops._C.kimi_k3_attn_res( + prefix, + delta, + blocks, + norm_weight, + qk_weight, + output_norm_weight, + output, + num_blocks, + eps, + output_norm_eps, + ) + return output + + def concat_and_cache_mla_rope_fused( positions: torch.Tensor, q_pe: torch.Tensor, diff --git a/vllm/models/kimi_k3/amd/__init__.py b/vllm/models/kimi_k3/amd/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/kimi_k3/amd/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/kimi_k3/amd/ops/__init__.py b/vllm/models/kimi_k3/amd/ops/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/models/kimi_k3/amd/ops/attn_res.py b/vllm/models/kimi_k3/amd/ops/attn_res.py new file mode 100644 index 0000000000..c00a3422ca --- /dev/null +++ b/vllm/models/kimi_k3/amd/ops/attn_res.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li +# +# This file contains code adapted from the flash-linear-attention project. +# The original source code was licensed under the MIT license and included +# the following copyright notice: +# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li + +import torch + +from vllm.triton_utils import tl, triton + + +@triton.jit +def _attn_res_kernel( + prefix_ptr, + blocks_ptr, + norm_weight_ptr, + qk_weight_ptr, + output_ptr, + stride_prefix_m: tl.constexpr, + stride_block_m: tl.constexpr, + stride_block_r: tl.constexpr, + stride_output_m: tl.constexpr, + num_blocks: tl.constexpr, + hidden_size: tl.constexpr, + eps: tl.constexpr, + BLOCK_L: tl.constexpr, + BLOCK_D: tl.constexpr, +): + row_idx = tl.program_id(0).to(tl.int64) + d_offsets = tl.max_contiguous(tl.arange(0, BLOCK_D), BLOCK_D) + d_mask = d_offsets < hidden_size + + prefix = tl.load( + prefix_ptr + row_idx * stride_prefix_m + d_offsets, + mask=d_mask, + other=0.0, + ).to(tl.float32) + input_qk_weight = tl.load(norm_weight_ptr + d_offsets, mask=d_mask, other=0.0).to( + tl.float32 + ) * tl.load(qk_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(tl.float32) + + max_logit = tl.full((), -float("inf"), tl.float32) + denominator = tl.zeros((), tl.float32) + mixed = tl.zeros((BLOCK_D,), tl.float32) + num_sources = num_blocks + 1 + + for source_tile in range(tl.cdiv(num_sources, BLOCK_L)): + source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L) + source_mask = source_offsets < num_sources + is_prefix = source_offsets == num_blocks + block_ptrs = ( + blocks_ptr + + row_idx * stride_block_m + + source_offsets[:, None] * stride_block_r + + d_offsets[None, :] + ) + block_values = tl.load( + block_ptrs, + mask=(source_mask[:, None] & ~is_prefix[:, None] & d_mask[None, :]), + other=0.0, + eviction_policy="evict_first", + ).to(tl.float32) + values = tl.where(is_prefix[:, None], prefix[None, :], block_values) + reciprocal_std = tl.rsqrt( + tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps + ) + logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std + scores = tl.where(source_mask, logits, -float("inf")) + + new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0)) + old_scale = tl.exp(max_logit - new_max_logit) + block_scales = tl.exp(scores - new_max_logit) + denominator = denominator * old_scale + tl.sum(block_scales, axis=0) + mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0) + max_logit = new_max_logit + + output = mixed / denominator + tl.store( + output_ptr + row_idx * stride_output_m + d_offsets, + output, + mask=d_mask, + ) + + +def attn_res( + prefix: torch.Tensor, + blocks: torch.Tensor, + norm_weight: torch.Tensor, + qk_weight: torch.Tensor, + num_blocks: int, + eps: float, +) -> torch.Tensor: + num_tokens, hidden_size = prefix.shape + assert 0 < num_blocks <= blocks.shape[1] + assert blocks.shape[0] == num_tokens + assert norm_weight.numel() == hidden_size + assert qk_weight.numel() == hidden_size + assert prefix.stride(-1) == 1 + assert blocks.stride(-1) == 1 + assert norm_weight.stride(-1) == 1 + assert qk_weight.stride(-1) == 1 + + output = prefix.new_empty(prefix.shape) + if num_tokens == 0: + return output + + if num_tokens >= 256 or num_blocks <= 1: + block_l, num_warps = 1, 4 + else: + block_l, num_warps = 4, 8 + _attn_res_kernel[(num_tokens,)]( + prefix, + blocks, + norm_weight, + qk_weight, + output, + prefix.stride(0), + blocks.stride(0), + blocks.stride(1), + output.stride(0), + num_blocks, + hidden_size, + eps, + BLOCK_L=block_l, + BLOCK_D=triton.next_power_of_2(hidden_size), + num_warps=num_warps, + num_stages=2, + ) + return output diff --git a/vllm/models/kimi_k3/nvidia/__init__.py b/vllm/models/kimi_k3/nvidia/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/kimi_k3/nvidia/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/kimi_k3/nvidia/ops/__init__.py b/vllm/models/kimi_k3/nvidia/ops/__init__.py new file mode 100644 index 0000000000..bbaee887ba --- /dev/null +++ b/vllm/models/kimi_k3/nvidia/ops/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from .attn_res import attn_res + +__all__ = ["attn_res"] diff --git a/vllm/models/kimi_k3/nvidia/ops/attn_res.py b/vllm/models/kimi_k3/nvidia/ops/attn_res.py new file mode 100644 index 0000000000..01078d6c01 --- /dev/null +++ b/vllm/models/kimi_k3/nvidia/ops/attn_res.py @@ -0,0 +1,245 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li +# +# This file contains code adapted from the flash-linear-attention project. +# The original source code was licensed under the MIT license and included +# the following copyright notice: +# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li + + +import torch + +from vllm import _custom_ops as ops +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton + + +# Consumed by kimi_k3_triton_warmup.py during kernel_warmup(). +def get_attn_res_triton_warmup_profiles( + max_blocks: int, +) -> tuple[tuple[int, bool, int, bool], ...]: + """Return the small-batch profiles that bypass the native kernel.""" + profiles = [ + (num_blocks, False, -1, True) for num_blocks in range(2, max_blocks + 1) + ] + profiles.extend( + (block_write_idx, True, block_write_idx, True) + for block_write_idx in range(2, max_blocks) + ) + profiles.append((max_blocks, True, -1, False)) + return tuple(profiles) + + +@triton.jit +def _attn_res_kernel( + prefix_ptr, + delta_ptr, + blocks_ptr, + norm_weight_ptr, + qk_weight_ptr, + output_norm_weight_ptr, + output_ptr, + stride_prefix_m: tl.constexpr, + stride_delta_m: tl.constexpr, + stride_block_m: tl.constexpr, + stride_block_r: tl.constexpr, + stride_output_m: tl.constexpr, + num_blocks: tl.constexpr, + hidden_size: tl.constexpr, + block_write_idx: tl.constexpr, + eps: tl.constexpr, + output_norm_eps: tl.constexpr, + HAS_DELTA: tl.constexpr, + WRITE_BLOCK: tl.constexpr, + APPLY_OUTPUT_NORM: tl.constexpr, + BLOCK_L: tl.constexpr, + BLOCK_D: tl.constexpr, + launch_pdl: tl.constexpr, +): + row_idx = tl.program_id(0).to(tl.int64) + d_offsets = tl.max_contiguous(tl.arange(0, BLOCK_D), BLOCK_D) + d_mask = d_offsets < hidden_size + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + updated_prefix = tl.load( + prefix_ptr + row_idx * stride_prefix_m + d_offsets, + mask=d_mask, + other=0.0, + ).to(tl.float32) + if HAS_DELTA: + delta = tl.load( + delta_ptr + row_idx * stride_delta_m + d_offsets, + mask=d_mask, + other=0.0, + ).to(tl.float32) + updated_prefix += delta + # Match the BF16 prefix-add result before using it as a residual source. + updated_prefix = updated_prefix.to(prefix_ptr.dtype.element_ty).to(tl.float32) + tl.store( + prefix_ptr + row_idx * stride_prefix_m + d_offsets, + updated_prefix, + mask=d_mask, + ) + if WRITE_BLOCK: + tl.store( + blocks_ptr + + row_idx * stride_block_m + + block_write_idx * stride_block_r + + d_offsets, + updated_prefix, + mask=d_mask, + ) + # With only the prefix source, the AttnRes softmax is exactly one. + if num_blocks == 0: + mixed = updated_prefix + else: + # Reloading avoids keeping the full prefix vector live across the loop. + if HAS_DELTA: + tl.debug_barrier() + input_qk_weight = tl.load( + norm_weight_ptr + d_offsets, mask=d_mask, other=0.0 + ).to(tl.float32) * tl.load( + qk_weight_ptr + d_offsets, mask=d_mask, other=0.0 + ).to(tl.float32) + max_logit = tl.full((), -float("inf"), tl.float32) + denominator = tl.zeros((), tl.float32) + mixed = tl.zeros((BLOCK_D,), tl.float32) + + num_sources = num_blocks + 1 + for source_tile in range(tl.cdiv(num_sources, BLOCK_L)): + source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L) + source_mask = source_offsets < num_sources + is_prefix = source_offsets == num_blocks + block_ptrs = ( + blocks_ptr + + row_idx * stride_block_m + + source_offsets[:, None] * stride_block_r + + d_offsets[None, :] + ) + prefix_ptrs = ( + prefix_ptr + + row_idx * stride_prefix_m + + source_offsets[:, None] * 0 + + d_offsets[None, :] + ) + value_ptrs = tl.where(is_prefix[:, None], prefix_ptrs, block_ptrs) + values = tl.load( + value_ptrs, + mask=source_mask[:, None] & d_mask[None, :], + other=0.0, + eviction_policy="evict_first", + ).to(tl.float32) + reciprocal_std = tl.rsqrt( + tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps + ) + logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std + scores = tl.where(source_mask, logits, -float("inf")) + + new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0)) + old_scale = tl.exp(max_logit - new_max_logit) + block_scales = tl.exp(scores - new_max_logit) + denominator = denominator * old_scale + tl.sum(block_scales, axis=0) + mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0) + max_logit = new_max_logit + + mixed /= denominator + output = mixed + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + + if APPLY_OUTPUT_NORM: + output_reciprocal_std = tl.rsqrt( + tl.sum(tl.where(d_mask, mixed * mixed, 0.0), axis=0) * (1.0 / hidden_size) + + output_norm_eps + ) + output_norm_weight = tl.load( + output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0 + ).to(tl.float32) + output = mixed * output_reciprocal_std * output_norm_weight + tl.store( + output_ptr + row_idx * stride_output_m + d_offsets, + output, + mask=d_mask, + ) + + +def attn_res( + prefix: torch.Tensor, + delta: torch.Tensor | None, + blocks: torch.Tensor, + norm_weight: torch.Tensor, + qk_weight: torch.Tensor, + output_norm_weight: torch.Tensor | None, + num_blocks: int, + block_write_idx: int, + eps: float, + output_norm_eps: float, +) -> torch.Tensor: + num_tokens, hidden_size = prefix.shape + assert prefix.stride(-1) == 1 + assert delta is None or delta.stride(-1) == 1 + assert blocks.stride(-1) == 1 + assert norm_weight.stride(-1) == 1 + assert qk_weight.stride(-1) == 1 + assert output_norm_weight is None or output_norm_weight.stride(-1) == 1 + # The in-tree NVIDIA kernel covers the common fused-add + output-norm path; + # Triton handles block boundaries and final pre-norm output. + if ( + hidden_size == 7168 + and delta is not None + and output_norm_weight is not None + and num_blocks > 0 + and block_write_idx < 0 + and current_platform.is_device_capability_family(100) + ): + return ops.kimi_k3_attn_res( + prefix, + delta, + blocks, + norm_weight, + qk_weight, + output_norm_weight, + num_blocks, + eps, + output_norm_eps, + ) + output = prefix.new_empty(prefix.shape) + # Tuned on GB300: source tiling helps decode, while one-source tiles scale + # better for prefill. + # Keep get_attn_res_triton_warmup_profiles in sync with these fallbacks. + if num_tokens >= 256 or num_blocks <= 1: + block_l, num_warps = 1, 4 + else: + block_l, num_warps = 4, 8 + _attn_res_kernel[(num_tokens,)]( + prefix, + delta, + blocks, + norm_weight, + qk_weight, + output_norm_weight, + output, + prefix.stride(0), + 0 if delta is None else delta.stride(0), + blocks.stride(0), + blocks.stride(1), + output.stride(0), + num_blocks, + hidden_size, + block_write_idx, + eps, + output_norm_eps, + HAS_DELTA=delta is not None, + WRITE_BLOCK=block_write_idx >= 0, + APPLY_OUTPUT_NORM=output_norm_weight is not None, + BLOCK_L=block_l, + BLOCK_D=triton.next_power_of_2(hidden_size), + num_warps=num_warps, + num_stages=2, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return output