[Kernel]Add norm fused kernel for Step-5 Preview and Muse Spark Open (#62132)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-10-17 01:53:00 -07:00
committed by GitHub
parent eb8de8d1a1
commit e56b1603af
10 changed files with 2461 additions and 0 deletions
@@ -0,0 +1,752 @@
/*
* SPDX-License-Identifier: Apache-2.0
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
*
* Fused Muse Spark attention pre-processing: weightless QK RMSNorm, GPT-J
* interleaved RoPE (skipped on NoPE layers, ``cos_sin_cache == None``) and
* the K/V insert of ``reshape_and_cache_flash``, over ``[q | k | v | o_gate]``
* rows.
*
* A group of ``head_dim * sizeof(scalar_t) / 16`` lanes owns one head (8 lanes
* for bf16 head_dim=64), so each lane moves one 16-byte vector, the rotated
* pair (2i, 2i+1) is lane-local, and the norm reduction is 3 shuffles.
*
* Slot enumeration per token:
* [0, nq) Q heads -> norm (+RoPE), in place or to
* ``q_fp8_out``
* [nq, nq+nkv) K heads -> norm (+RoPE), in place and into
* the K cache
* [nq+nkv, nq+2*nkv) V heads -> copied into the V cache (no norm)
* V slots exist only when a KV cache is bound.
*
* Norm and rotation run in fp32 and round once at the store. Both fp8 paths
* quantize after rounding to the model dtype, the same boundary
* ``reshape_and_cache_flash`` and ``scaled_fp8_quant`` use.
*/
#include <algorithm>
#include <cmath>
#include <cuda_runtime.h>
#include <type_traits>
#include "torch_utils.h"
#include "core/cuda_compat.h"
#include "type_convert.cuh"
#include "core/vec_dtypes/dtype_fp8.cuh"
#include "dispatch_utils.h"
#ifndef USE_ROCM
#include <cuda_fp8.h>
#include "quantization/w8a8/fp8/nvidia/quant_utils.cuh"
#else
#include <hip/hip_fp8.h>
#include "quantization/w8a8/fp8/amd/quant_utils.cuh"
#endif
#ifndef FINAL_MASK
#ifdef USE_ROCM
#define FINAL_MASK 0xffffffffffffffffULL
#else
#define FINAL_MASK 0xffffffffu
#endif
#endif
namespace vllm {
namespace muse_spark_fused_ops {
namespace {
inline int getSMVersion() {
auto* props = get_device_prop();
return props->major * 10 + props->minor;
}
inline int getSMCount() {
auto* props = get_device_prop();
return props->multiProcessorCount;
}
} // namespace
// 16 bytes per lane: the widest vector access, and the reason a head is owned
// by a group of lanes rather than a whole warp.
constexpr int kBytesPerLane = 16;
#ifdef USE_ROCM
using WarpMask = uint64_t;
constexpr unsigned kWarpSize = 64;
#else
using WarpMask = unsigned;
constexpr unsigned kWarpSize = 32;
#endif
// Slot -> (token, head) needs a division by a value only known at launch. The
// kernel is issue-bound, and a 32-bit hardware division is ~20 instructions of
// it, so the reciprocal is computed once on the host instead.
struct FastDiv {
uint32_t mul;
uint32_t shr;
int div;
};
inline FastDiv makeFastDiv(int denom) {
// The reciprocal form needs denom >= 2; denom == 1 is carried as a flag the
// device side selects on. Dividends stay below 2^31, which covers every slot
// index this kernel can form.
if (denom <= 1) return FastDiv{0u, 0u, 1};
int lg = 31 - __builtin_clz(denom);
if ((1 << lg) < denom) lg++;
uint32_t const p = 31 + static_cast<uint32_t>(lg);
uint64_t const m = ((1ull << p) + static_cast<uint64_t>(denom) - 1) /
static_cast<uint64_t>(denom);
return FastDiv{static_cast<uint32_t>(m), p - 32, denom};
}
__device__ __forceinline__ void fastDivmod(int& quo, int& rem, int src,
FastDiv const fd) {
int const q =
static_cast<int>(__umulhi(static_cast<uint32_t>(src), fd.mul) >> fd.shr);
// A select, not a branch: fd.div is uniform across the launch.
quo = (fd.div == 1) ? src : q;
rem = src - quo * fd.div;
}
// Sum across the LANES contiguous lanes that share one head. The group is
// LANES-aligned inside the warp, so an XOR butterfly over masks < LANES never
// leaves it. `group_mask` names exactly the group: whole groups are uniformly
// active or inactive, so a warp whose other groups bailed out still syncs.
template <int LANES>
__device__ __forceinline__ float groupReduceSum(float val,
WarpMask const group_mask) {
#pragma unroll
for (int mask = LANES >> 1; mask > 0; mask >>= 1) {
val += __shfl_xor_sync(group_mask, val, mask, 32);
}
return val;
}
// 16 contiguous bytes -> kElems fp32 registers, keeping the raw vector so a
// pass-through slot (V) can be re-stored without a conversion round trip.
template <typename scalar_t, int kElems>
__device__ __forceinline__ uint4 loadElems(scalar_t const* __restrict__ src,
float (&elems)[kElems]) {
using Converter = vllm::_typeConvert<scalar_t>;
using T2 = typename Converter::packed_hip_type;
uint4 raw = *reinterpret_cast<uint4 const*>(src);
auto const* p = reinterpret_cast<T2 const*>(&raw);
#pragma unroll
for (int i = 0; i < kElems / 2; i++) {
float2 f2 = Converter::convert(p[i]);
elems[2 * i] = f2.x;
elems[2 * i + 1] = f2.y;
}
return raw;
}
// The lane's slice of the rope table: kPairs cos and kPairs sin values, each
// run contiguous, so each is one vector load (16 bytes for the fp32 table, 8
// for a model-dtype one) instead of kPairs scalar ones. rotary_dim being a
// multiple of 16 is what keeps both runs naturally aligned.
template <typename rope_t, int kPairs>
__device__ __forceinline__ void loadRopeVec(rope_t const* __restrict__ cos_ptr,
rope_t const* __restrict__ sin_ptr,
float (&c)[kPairs],
float (&s)[kPairs]) {
static_assert(kPairs == 4, "rope vector load assumes 4 pairs per lane");
if constexpr (std::is_same_v<rope_t, float>) {
float4 const cv = VLLM_LDG(reinterpret_cast<float4 const*>(cos_ptr));
float4 const sv = VLLM_LDG(reinterpret_cast<float4 const*>(sin_ptr));
c[0] = cv.x, c[1] = cv.y, c[2] = cv.z, c[3] = cv.w;
s[0] = sv.x, s[1] = sv.y, s[2] = sv.z, s[3] = sv.w;
} else {
using Converter = vllm::_typeConvert<rope_t>;
using T2 = typename Converter::packed_hip_type;
uint2 const cv = VLLM_LDG(reinterpret_cast<uint2 const*>(cos_ptr));
uint2 const sv = VLLM_LDG(reinterpret_cast<uint2 const*>(sin_ptr));
auto const* cp = reinterpret_cast<T2 const*>(&cv);
auto const* sp = reinterpret_cast<T2 const*>(&sv);
#pragma unroll
for (int i = 0; i < kPairs / 2; i++) {
float2 const cf = Converter::convert(cp[i]);
float2 const sf = Converter::convert(sp[i]);
c[2 * i] = cf.x, c[2 * i + 1] = cf.y;
s[2 * i] = sf.x, s[2 * i + 1] = sf.y;
}
}
}
// kElems fp32 registers -> the 16-byte scalar_t vector they round to. Returned
// rather than stored so the fp8 cache path can reuse the *rounded* values,
// which is what reshape_and_cache_flash quantizes from.
template <typename scalar_t, int kElems>
__device__ __forceinline__ uint4 packElems(float const (&elems)[kElems]) {
using Converter = vllm::_typeConvert<scalar_t>;
using T2 = typename Converter::packed_hip_type;
uint4 raw;
auto* p = reinterpret_cast<T2*>(&raw);
#pragma unroll
for (int i = 0; i < kElems / 2; i++) {
p[i] = Converter::convert(make_float2(elems[2 * i], elems[2 * i + 1]));
}
return raw;
}
// Scatter one lane's 16 bytes into the KV cache. kAuto stores the model dtype
// verbatim; the fp8 dtypes quantize the value *after* it has been rounded to
// scalar_t, which is the boundary reshape_and_cache_flash uses.
//
// `inv_scale` is taken once per thread: per-element fp32 divisions would cost
// more than the rest of the kernel. The extra rounding is 2^-24 relative.
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt,
int kElems>
__device__ __forceinline__ void storeCache(cache_t* __restrict__ dst,
uint4 const& packed,
float const inv_scale) {
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
*reinterpret_cast<uint4*>(dst) = packed;
} else {
using Converter = vllm::_typeConvert<scalar_t>;
using HipT = typename Converter::hip_type;
auto const* rounded = reinterpret_cast<HipT const*>(&packed);
#ifndef USE_ROCM
constexpr __nv_fp8_interpretation_t kInterp =
(kv_dt == Fp8KVCacheDataType::kFp8E4M3) ? __NV_E4M3 : __NV_E5M2;
__nv_fp8x2_storage_t out[kElems / 2];
#pragma unroll
for (int i = 0; i < kElems / 2; i++) {
float2 v =
make_float2(Converter::convert(rounded[2 * i]) * inv_scale,
Converter::convert(rounded[2 * i + 1]) * inv_scale);
out[i] = __nv_cvt_float2_to_fp8x2(v, __NV_SATFINITE, kInterp);
}
// kElems fp8 bytes: 8 for the 2-byte dtypes this kernel serves.
static_assert(kElems == 8, "fp8 store assumes 8 elements per lane");
*reinterpret_cast<uint2*>(dst) = *reinterpret_cast<uint2 const*>(out);
#else
#pragma unroll
for (int i = 0; i < kElems; i++) {
float const v = Converter::convert(rounded[i]) * inv_scale;
dst[i] = fp8::scaled_convert<cache_t, float, kv_dt>(v, 1.0f);
}
#endif
}
}
// Grid is 1D over (token, head-chunk) pairs. A chunk is HEADS_PER_GROUP
// consecutive head slots of one token, served by kLanes contiguous lanes; the
// whole chunk shares the token's position and its slice of the rope table, so
// the slot division, the position load and the cos/sin load are paid once per
// chunk. `slots_per_token % HEADS_PER_GROUP == 0` keeps a chunk inside one
// token. `rope_t` is the cos/sin cache dtype, independent of the qkv dtype.
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt,
typename rope_t, int HEAD_DIM, int HEADS_PER_GROUP, bool kDoRope,
bool kInsertKV>
__global__
__launch_bounds__(256, 6) void fusedMuseSparkQKNormRopeKVInsertKernel(
scalar_t* __restrict__ qkv, // [N, qkv_stride] in/out
rope_t const* __restrict__ cos_sin_cache, // [max_pos, rotary_dim]
int64_t const* __restrict__ positions, // [N] i64
int64_t const* __restrict__ slot_mapping, // [N] i64 or nullptr
cache_t* __restrict__ kv_cache, // [nb, nkv, bs, 2*HEAD_DIM]
float const* __restrict__ k_scale, // [1] or nullptr
float const* __restrict__ v_scale, // [1] or nullptr
uint8_t* __restrict__ q_fp8_out, // [N, nq * HEAD_DIM] or nullptr
float const* __restrict__ q_scale, // [1] or nullptr
float const eps, int const rotary_dim, int const num_tokens, int const nq,
int const nkv, FastDiv const chunks_div, int64_t const qkv_stride,
FastDiv const block_div, int64_t const kv_s_block, int64_t const kv_s_head,
int64_t const kv_s_token) {
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
// _typeConvert<BFloat16> is unavailable on pre-Ampere; this kernel only ever
// runs with bf16/fp16 inputs. Discard the bf16 body there.
if constexpr (std::is_same_v<scalar_t, c10::BFloat16>) {
return;
} else {
#endif
constexpr int kElems = kBytesPerLane / sizeof(scalar_t);
constexpr int kLanes = HEAD_DIM / kElems;
constexpr int kPairs = kElems / 2;
constexpr int kH = HEADS_PER_GROUP;
static_assert(kLanes > 0 && kLanes <= 32 && (32 % kLanes) == 0,
"head_dim must give a power-of-two lane group <= 32");
int const tid = blockIdx.x * blockDim.x + threadIdx.x;
int const groupIdx = tid / kLanes;
int const dimBase = (tid % kLanes) * kElems;
int tokenIdx, chunk;
fastDivmod(tokenIdx, chunk, groupIdx, chunks_div);
int const firstSlot = chunk * kH;
// Out-of-range groups must not exit before the reductions below: the XOR
// butterfly needs every lane the mask names, and the mask covers the whole
// group. They ride along and skip every memory access instead.
bool const active = tokenIdx < num_tokens;
constexpr WarpMask kGroupBits =
(kLanes == 32) ? static_cast<WarpMask>(0xffffffffu)
: ((WarpMask{1} << kLanes) - WarpMask{1});
unsigned const group_shift =
(threadIdx.x & (kWarpSize - 1)) & ~static_cast<unsigned>(kLanes - 1);
WarpMask const group_mask = kGroupBits << group_shift;
// PDL: wait for the qkv-projection GEMM that produces `qkv`. No-op when
// PDL is not enabled on the launch.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaGridDependencySynchronize();
#endif
// Q, K and V are contiguous head slots at the head of the row, so the slot
// index doubles as the head offset. The trailing o-gate is skipped by the
// row stride alone.
scalar_t* const base = qkv + static_cast<int64_t>(tokenIdx) * qkv_stride +
static_cast<int64_t>(firstSlot) * HEAD_DIM + dimBase;
// Issue every load in the chunk before touching any of it, so the group has
// kH loads in flight instead of one. An inactive group leaves these unset;
// nothing downstream reads them, since every store is gated on `active`.
uint4 raw[kH];
if (active) {
#pragma unroll
for (int h = 0; h < kH; h++) {
raw[h] = *reinterpret_cast<uint4 const*>(base + h * HEAD_DIM);
}
}
float cosv[kPairs], sinv[kPairs];
bool const laneRotates = kDoRope && dimBase < rotary_dim;
if constexpr (kDoRope) {
if (active && laneRotates) {
int64_t const pos = positions[tokenIdx];
rope_t const* cache_ptr = cos_sin_cache + pos * rotary_dim;
loadRopeVec<rope_t, kPairs>(cache_ptr + dimBase / 2,
cache_ptr + rotary_dim / 2 + dimBase / 2,
cosv, sinv);
}
}
int64_t sm = -1;
if constexpr (kInsertKV) {
if (active) sm = slot_mapping[tokenIdx];
}
int cache_block = 0, cache_tok = 0;
float k_inv = 1.0f, v_inv = 1.0f;
float const q_inv = q_fp8_out ? 1.0f / *q_scale : 1.0f;
if constexpr (kInsertKV) {
if (sm >= 0) {
fastDivmod(cache_block, cache_tok, static_cast<int>(sm), block_div);
}
if constexpr (kv_dt != Fp8KVCacheDataType::kAuto) {
k_inv = 1.0f / *k_scale;
v_inv = 1.0f / *v_scale;
}
}
#pragma unroll
for (int h = 0; h < kH; h++) {
int const slot = firstSlot + h;
bool const isV = kInsertKV && slot >= nq + nkv;
bool const isK = slot >= nq && slot < nq + nkv;
bool const isQK = active && !isV;
float elems[kElems];
uint4 packed = raw[h];
if (isQK) {
using Converter = vllm::_typeConvert<scalar_t>;
using T2 = typename Converter::packed_hip_type;
auto const* p = reinterpret_cast<T2 const*>(&raw[h]);
#pragma unroll
for (int i = 0; i < kElems / 2; i++) {
float2 const f2 = Converter::convert(p[i]);
elems[2 * i] = f2.x;
elems[2 * i + 1] = f2.y;
}
float sumsq = 0.0f;
#pragma unroll
for (int i = 0; i < kElems; i++) sumsq += elems[i] * elems[i];
sumsq = groupReduceSum<kLanes>(sumsq, group_mask);
float const rms_rcp =
rsqrtf(sumsq / static_cast<float>(HEAD_DIM) + eps);
#pragma unroll
for (int i = 0; i < kElems; i++) elems[i] *= rms_rcp;
if constexpr (kDoRope) {
if (laneRotates) {
#pragma unroll
for (int i = 0; i < kPairs; i++) {
float const v0 = elems[2 * i];
float const v1 = elems[2 * i + 1];
elems[2 * i] = v0 * cosv[i] - v1 * sinv[i];
elems[2 * i + 1] = v0 * sinv[i] + v1 * cosv[i];
}
}
}
// Rounding to scalar_t happens exactly once, here, and both the
// Q output and the fp8 cache quantization consume the same
// rounded value. V passes through with the vector it was loaded with.
packed = packElems<scalar_t, kElems>(elems);
if (q_fp8_out && slot < nq) {
int64_t const q_off =
(static_cast<int64_t>(tokenIdx) * nq + slot) * HEAD_DIM + dimBase;
storeCache<scalar_t, uint8_t, Fp8KVCacheDataType::kFp8E4M3, kElems>(
q_fp8_out + q_off, packed, q_inv);
} else {
*reinterpret_cast<uint4*>(base + h * HEAD_DIM) = packed;
}
}
if constexpr (kInsertKV) {
if (sm >= 0 && (isK || isV)) { // sm < 0 marks a padded token
int const head = isK ? (slot - nq) : (slot - nq - nkv);
int64_t const off = static_cast<int64_t>(cache_block) * kv_s_block +
head * kv_s_head +
static_cast<int64_t>(cache_tok) * kv_s_token +
(isK ? 0 : HEAD_DIM) + dimBase;
storeCache<scalar_t, cache_t, kv_dt, kElems>(kv_cache + off, packed,
isK ? k_inv : v_inv);
}
}
}
// PDL: let a dependent successor launch early.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
#endif
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
}
#endif
}
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt,
typename rope_t, int HEAD_DIM, int HEADS_PER_GROUP>
void launchFusedMuseSparkTyped(
scalar_t* qkv, rope_t const* cos_sin_cache, int64_t const* positions,
int64_t const* slot_mapping, cache_t* kv_cache, float const* k_scale,
float const* v_scale, uint8_t* q_fp8_out, float const* q_scale,
float const eps, int const rotary_dim, int const num_tokens, int const nq,
int const nkv, int64_t const qkv_stride, int const block_size,
int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token,
bool const do_rope, bool const insert_kv, cudaStream_t stream) {
FastDiv const block_div = makeFastDiv(block_size);
constexpr int kElems = kBytesPerLane / sizeof(scalar_t);
constexpr int kLanes = HEAD_DIM / kElems;
int const slots_per_token = nq + nkv + (insert_kv ? nkv : 0);
int const chunks_per_token = slots_per_token / HEADS_PER_GROUP;
FastDiv const chunks_div = makeFastDiv(chunks_per_token);
int64_t const total_lanes =
static_cast<int64_t>(num_tokens) * chunks_per_token * kLanes;
if (total_lanes == 0) return;
constexpr int kBlockSize = 256;
int const grid =
static_cast<int>((total_lanes + kBlockSize - 1) / kBlockSize);
#ifndef USE_ROCM
// PDL wherever the hardware supports it (SM90+); a plain launch elsewhere.
static int const sm_version = getSMVersion();
cudaLaunchConfig_t config;
config.gridDim = dim3(grid);
config.blockDim = dim3(kBlockSize);
config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.attrs = attrs;
config.numAttrs = (sm_version >= 90) ? 1 : 0;
#define LAUNCH_MUSE(DO_ROPE, INSERT) \
cudaLaunchKernelEx(&config, \
fusedMuseSparkQKNormRopeKVInsertKernel< \
scalar_t, cache_t, kv_dt, rope_t, HEAD_DIM, \
HEADS_PER_GROUP, DO_ROPE, INSERT>, \
qkv, cos_sin_cache, positions, slot_mapping, kv_cache, \
k_scale, v_scale, q_fp8_out, q_scale, eps, rotary_dim, \
num_tokens, nq, nkv, chunks_div, qkv_stride, block_div, \
kv_s_block, kv_s_head, kv_s_token)
#else
// clang-format off
#define LAUNCH_MUSE(DO_ROPE, INSERT) \
fusedMuseSparkQKNormRopeKVInsertKernel< \
scalar_t, cache_t, kv_dt, rope_t, HEAD_DIM, HEADS_PER_GROUP, DO_ROPE, \
INSERT><<<grid, kBlockSize, 0, stream>>>( \
qkv, cos_sin_cache, positions, slot_mapping, kv_cache, k_scale, \
v_scale, q_fp8_out, q_scale, eps, rotary_dim, num_tokens, nq, nkv, \
chunks_div, qkv_stride, block_div, kv_s_block, kv_s_head, kv_s_token)
// clang-format on
#endif
if (do_rope) {
if (insert_kv) {
LAUNCH_MUSE(true, true);
} else {
LAUNCH_MUSE(true, false);
}
} else {
if (insert_kv) {
LAUNCH_MUSE(false, true);
} else {
LAUNCH_MUSE(false, false);
}
}
#undef LAUNCH_MUSE
}
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
void launchFusedMuseSpark(
scalar_t* qkv, void const* cos_sin_cache, bool const rope_is_fp32,
int64_t const* positions, int64_t const* slot_mapping, cache_t* kv_cache,
float const* k_scale, float const* v_scale, uint8_t* q_fp8_out,
float const* q_scale, float const eps, int const rotary_dim,
int const num_tokens, int const nq, int const nkv, int const head_dim,
int64_t const qkv_stride, int const block_size, int64_t const kv_s_block,
int64_t const kv_s_head, int64_t const kv_s_token, bool const do_rope,
bool const insert_kv, cudaStream_t stream) {
// The group width must divide the slot count. Widen only while the grid
// still covers a full wave; decode-sized batches keep the narrow schedule.
int const slots_per_token = nq + nkv + (insert_kv ? nkv : 0);
constexpr int kElemsSel = kBytesPerLane / sizeof(scalar_t);
constexpr int kBlockSel = 256;
int const lanes_per_head = head_dim / kElemsSel;
int64_t const full_wave =
static_cast<int64_t>(getSMCount()) * (2048 / kBlockSel);
int heads_per_group = 1;
for (int h : {4, 2}) {
if (slots_per_token % h != 0) continue;
int64_t const blocks = (static_cast<int64_t>(num_tokens) *
(slots_per_token / h) * lanes_per_head +
kBlockSel - 1) /
kBlockSel;
if (blocks >= full_wave) {
heads_per_group = h;
break;
}
}
// TP4/TP8 amortize head addressing before filling a full CTA wave. RoPE
// benefits earlier; FP8 conversion keeps wider groups expensive for longer.
if (insert_kv && head_dim == 64 &&
((nq == 16 && nkv == 2) || (nq == 32 && nkv == 4))) {
int64_t const query_heads = static_cast<int64_t>(num_tokens) * nq;
if (do_rope && query_heads >= 24576) {
heads_per_group = std::max(heads_per_group, 2);
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
if (query_heads >= 49152) heads_per_group = 4;
}
} else if (!do_rope && query_heads >= 32768) {
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
heads_per_group = std::max(heads_per_group, 2);
}
}
}
#define DISPATCH_ARGS(RT, HD, HPG) \
launchFusedMuseSparkTyped<scalar_t, cache_t, kv_dt, RT, HD, HPG>( \
qkv, reinterpret_cast<RT const*>(cos_sin_cache), positions, \
slot_mapping, kv_cache, k_scale, v_scale, q_fp8_out, q_scale, eps, \
rotary_dim, num_tokens, nq, nkv, qkv_stride, block_size, kv_s_block, \
kv_s_head, kv_s_token, do_rope, insert_kv, stream)
#define DISPATCH_HPG(RT, HD) \
switch (heads_per_group) { \
case 4: \
DISPATCH_ARGS(RT, HD, 4); \
break; \
case 2: \
DISPATCH_ARGS(RT, HD, 2); \
break; \
default: \
DISPATCH_ARGS(RT, HD, 1); \
break; \
}
// A NoPE launch never reads the table, so it can share the fp32 branch.
#define DISPATCH_ROPE_T(HD) \
if (!do_rope || rope_is_fp32) { \
DISPATCH_HPG(float, HD); \
} else { \
DISPATCH_HPG(scalar_t, HD); \
}
switch (head_dim) {
case 64:
DISPATCH_ROPE_T(64);
break;
case 128:
DISPATCH_ROPE_T(128);
break;
case 256:
DISPATCH_ROPE_T(256);
break;
default:
STD_TORCH_CHECK(false, "unsupported head_dim: ", head_dim);
}
#undef DISPATCH_ROPE_T
#undef DISPATCH_HPG
#undef DISPATCH_ARGS
}
} // namespace muse_spark_fused_ops
} // namespace vllm
// clang-format off
#define CALL_FUSED_MUSE_SPARK(_RAW_T, CACHE_T, KV_DTYPE) \
vllm::muse_spark_fused_ops::launchFusedMuseSpark<st, CACHE_T, KV_DTYPE>( \
reinterpret_cast<st*>(qkv.data_ptr()), \
do_rope ? cos_sin_cache->data_ptr() : nullptr, rope_is_fp32, \
reinterpret_cast<int64_t const*>(positions.data_ptr()), \
insert_kv ? reinterpret_cast<int64_t const*>(slot_mapping->data_ptr()) \
: nullptr, \
insert_kv ? reinterpret_cast<CACHE_T*>(kv_cache->data_ptr()) : nullptr, \
needs_scales ? reinterpret_cast<float const*>(k_scale->data_ptr()) \
: nullptr, \
needs_scales ? reinterpret_cast<float const*>(v_scale->data_ptr()) \
: nullptr, \
q_fp8_out ? reinterpret_cast<uint8_t*>(q_fp8_out->data_ptr()) : nullptr, \
q_scale ? reinterpret_cast<float const*>(q_scale->data_ptr()) : nullptr, \
static_cast<float>(eps), static_cast<int>(rotary_dim), num_tokens, nq, \
nkv, hd, qkv_stride, static_cast<int>(block_size), kv_s_block, \
kv_s_head, kv_s_token, do_rope, insert_kv, stream)
// clang-format on
void fused_muse_spark_qknorm_rope_kv_insert(
torch::stable::Tensor& qkv, // [N, (nq + 2*nkv + gate)*hd]
torch::stable::Tensor const& positions, // [N] i64
int64_t num_heads, int64_t num_kv_heads, int64_t head_dim, double eps,
std::optional<torch::stable::Tensor> cos_sin_cache, // None => NoPE layer
int64_t rotary_dim,
std::optional<torch::stable::Tensor> slot_mapping, // [N] i64
std::optional<torch::stable::Tensor> kv_cache, // [nb, nkv, bs, 2*hd]
int64_t block_size, const std::string& kv_cache_dtype,
std::optional<torch::stable::Tensor> k_scale, // [1] fp32, fp8 cache only
std::optional<torch::stable::Tensor> v_scale,
std::optional<torch::stable::Tensor> q_fp8_out,
std::optional<torch::stable::Tensor> q_scale) {
STD_TORCH_CHECK(qkv.is_cuda() && qkv.dim() == 2 && qkv.stride(1) == 1,
"qkv must be a 2-D CUDA tensor with a contiguous row");
STD_TORCH_CHECK(
qkv.scalar_type() == torch::headeronly::ScalarType::Half ||
qkv.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"qkv must be float16 or bfloat16");
STD_TORCH_CHECK(
positions.is_cuda() &&
positions.scalar_type() == torch::headeronly::ScalarType::Long,
"positions must be int64 CUDA");
int const num_tokens = static_cast<int>(qkv.size(0));
int const nq = static_cast<int>(num_heads);
int const nkv = static_cast<int>(num_kv_heads);
int const hd = static_cast<int>(head_dim);
int64_t const qkv_stride = qkv.stride(0);
STD_TORCH_CHECK(q_fp8_out.has_value() == q_scale.has_value(),
"q_fp8_out and q_scale must be provided together");
if (q_fp8_out) {
STD_TORCH_CHECK(
q_fp8_out->is_cuda() &&
q_fp8_out->get_device_index() == qkv.get_device_index() &&
q_fp8_out->scalar_type() ==
torch::headeronly::ScalarType::Float8_e4m3fn &&
q_fp8_out->dim() == 2 && q_fp8_out->is_contiguous() &&
q_fp8_out->size(0) == num_tokens &&
q_fp8_out->size(1) == static_cast<int64_t>(nq) * hd,
"q_fp8_out must be contiguous E4M3 [num_tokens, num_heads * head_dim] "
"on the qkv device");
STD_TORCH_CHECK(
q_scale->is_cuda() &&
q_scale->get_device_index() == qkv.get_device_index() &&
q_scale->scalar_type() == torch::headeronly::ScalarType::Float &&
q_scale->numel() == 1,
"q_scale must be a single fp32 value on the qkv device");
}
STD_TORCH_CHECK(hd == 64 || hd == 128 || hd == 256,
"head_dim must be 64, 128 or 256");
STD_TORCH_CHECK(qkv.size(1) >= static_cast<int64_t>(nq + 2 * nkv) * hd,
"qkv row must hold at least [q|k|v]");
// A lane moves 16 bytes, so every row start and every head start must land
// on a 16-byte boundary. Every supported head_dim is a multiple of 8 at 2
// bytes, so heads are fine; the row stride is the one that can break it.
STD_TORCH_CHECK(qkv_stride % 8 == 0,
"qkv row stride must be a multiple of 8 elements, got ",
qkv_stride);
bool const do_rope = cos_sin_cache.has_value();
bool rope_is_fp32 = true;
if (do_rope) {
STD_TORCH_CHECK(cos_sin_cache->is_cuda() && cos_sin_cache->dim() == 2 &&
cos_sin_cache->is_contiguous(),
"cos_sin_cache must be a contiguous 2-D CUDA tensor");
STD_TORCH_CHECK(cos_sin_cache->size(1) == rotary_dim,
"cos_sin_cache shape must be [max_pos, rotary_dim]");
STD_TORCH_CHECK(
cos_sin_cache->scalar_type() == torch::headeronly::ScalarType::Float ||
cos_sin_cache->scalar_type() == qkv.scalar_type(),
"cos_sin_cache must be float32 or match the qkv dtype");
rope_is_fp32 =
cos_sin_cache->scalar_type() == torch::headeronly::ScalarType::Float;
// A lane owns 16 bytes = 8 elements of the head and must be wholly inside
// or wholly outside the rotated prefix.
STD_TORCH_CHECK(rotary_dim > 0 && rotary_dim <= hd && rotary_dim % 16 == 0,
"rotary_dim must be a positive multiple of 16 and <= "
"head_dim");
}
vllm::Fp8KVCacheDataType const kv_dt =
vllm::get_fp8_kv_cache_data_type(kv_cache_dtype);
bool const insert_kv = kv_cache.has_value() && kv_cache->numel() > 0;
bool const needs_scales =
insert_kv && kv_dt != vllm::Fp8KVCacheDataType::kAuto;
int64_t kv_s_block = 0, kv_s_head = 0, kv_s_token = 0;
if (insert_kv) {
STD_TORCH_CHECK(
slot_mapping.has_value() && slot_mapping->is_cuda() &&
slot_mapping->scalar_type() == torch::headeronly::ScalarType::Long,
"insert mode requires an int64 CUDA slot_mapping");
STD_TORCH_CHECK(slot_mapping->numel() >= num_tokens,
"slot_mapping must cover every token");
STD_TORCH_CHECK(kv_cache->dim() == 4 && kv_cache->stride(3) == 1,
"kv_cache must be [nb, nkv, bs, 2*head_dim] with a "
"contiguous content dim");
STD_TORCH_CHECK(kv_cache->size(3) == 2 * hd,
"kv_cache content dim must be 2*head_dim");
STD_TORCH_CHECK(kv_cache->size(1) == nkv,
"kv_cache head dim must equal num_kv_heads");
STD_TORCH_CHECK(kv_cache->size(2) == block_size,
"kv_cache token dim must equal block_size");
if (kv_dt == vllm::Fp8KVCacheDataType::kAuto) {
STD_TORCH_CHECK(kv_cache->scalar_type() == qkv.scalar_type(),
"auto kv_cache dtype must match qkv");
} else {
STD_TORCH_CHECK(
kv_cache->scalar_type() == torch::headeronly::ScalarType::Byte ||
kv_cache->scalar_type() ==
torch::headeronly::ScalarType::Float8_e4m3fn,
"fp8 kv_cache must use uint8 or float8_e4m3fn storage");
STD_TORCH_CHECK(k_scale.has_value() && v_scale.has_value(),
"fp8 kv cache requires k_scale and v_scale");
STD_TORCH_CHECK(
k_scale->numel() == 1 && v_scale->numel() == 1 &&
k_scale->scalar_type() == torch::headeronly::ScalarType::Float &&
v_scale->scalar_type() == torch::headeronly::ScalarType::Float,
"k_scale and v_scale must be single fp32 values");
}
kv_s_block = kv_cache->stride(0);
kv_s_head = kv_cache->stride(1);
kv_s_token = kv_cache->stride(2);
}
const torch::stable::accelerator::DeviceGuard device_guard(
qkv.get_device_index());
auto stream = get_current_cuda_stream(qkv.get_device_index());
VLLM_STABLE_DISPATCH_HALF_TYPES(
qkv.scalar_type(), "fused_muse_spark_qknorm_rope_kv_insert", [&] {
using st = scalar_t;
DISPATCH_BY_KV_CACHE_DTYPE(qkv.scalar_type(), kv_cache_dtype,
CALL_FUSED_MUSE_SPARK);
});
}
#undef CALL_FUSED_MUSE_SPARK
+567
View File
@@ -0,0 +1,567 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
// Step5 fused QK norm, RoPE, and KV-cache update kernel.
// This kernel is built into `_C_stable_libtorch` and uses the libtorch stable
// ABI (torch::stable::Tensor / STD_TORCH_CHECK) instead of ATen and c10.
#include "torch_utils.h"
#include "dispatch_utils.h"
#include "type_convert.cuh"
#include "core/vec_dtypes/dtype_fp8.cuh"
#include "quantization/w8a8/fp8/nvidia/quant_utils.cuh"
#include <torch/csrc/stable/macros.h>
#include <cuda_runtime.h>
#include <algorithm>
#include <type_traits>
namespace vllm {
namespace {
constexpr int kWarpSize = 32;
constexpr int kThreadsPerBlock = 128;
template <typename scalar_t, typename weight_t, typename rope_t, int HEAD_DIM,
bool WIDE_192_ROTARY = false, bool FP8_CACHE = false>
__global__ void qknorm_rope_cache_kernel(
const void* __restrict__ qkv_void, void* __restrict__ q_out_void,
void* __restrict__ k_out_void, void* __restrict__ v_out_void,
const void* __restrict__ q_weight_void,
const void* __restrict__ k_weight_void, const void* __restrict__ cos_void,
const void* __restrict__ sin_void, const int64_t* __restrict__ positions,
const int64_t* __restrict__ slot_mapping, void* __restrict__ k_cache_void,
void* __restrict__ v_cache_void, int64_t qkv_token_stride,
int64_t q_token_stride, int64_t k_token_stride, int64_t v_token_stride,
int64_t cos_token_stride, int64_t sin_token_stride,
int64_t cache_block_stride, int64_t cache_page_stride,
int64_t cache_head_stride, int num_tokens, int num_slots, int num_q_heads,
int num_kv_heads, int rotary_pairs, int block_size, float epsilon,
int block_size_shift, float norm_weight_bias, const float* k_scale,
const float* v_scale) {
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800
if constexpr (std::is_same_v<scalar_t, torch::headeronly::BFloat16> ||
std::is_same_v<weight_t, torch::headeronly::BFloat16> ||
std::is_same_v<rope_t, torch::headeronly::BFloat16>) {
return;
} else {
#endif
using ScalarConverter = _typeConvert<scalar_t>;
using WeightConverter = _typeConvert<weight_t>;
using RopeConverter = _typeConvert<rope_t>;
using Scalar = typename ScalarConverter::hip_type;
using Weight = typename WeightConverter::hip_type;
using Rope = typename RopeConverter::hip_type;
// Preserve the original per-head reduction and activation rounding.
static_assert(!WIDE_192_ROTARY || HEAD_DIM == 192);
constexpr int kThreadsPerHead =
WIDE_192_ROTARY ? 32
: (HEAD_DIM <= 64 ? 8 : (HEAD_DIM <= 192 ? 16 : 32));
constexpr int kValuesPerVector =
WIDE_192_ROTARY ? 8 : (HEAD_DIM == 192 ? 4 : 8);
constexpr int kValuesPerTile = kThreadsPerHead * kValuesPerVector;
constexpr int kNumVectors =
(HEAD_DIM + kValuesPerTile - 1) / kValuesPerTile;
constexpr int kValuesPerThread = kValuesPerVector * kNumVectors;
constexpr int kTileSize = kValuesPerTile * kNumVectors;
constexpr int kHeadsPerBlock = kThreadsPerBlock / kThreadsPerHead;
const auto* qkv = static_cast<const Scalar*>(qkv_void);
auto* q_out = static_cast<Scalar*>(q_out_void);
auto* k_out = static_cast<Scalar*>(k_out_void);
auto* v_out = static_cast<Scalar*>(v_out_void);
const auto* q_weight = static_cast<const Weight*>(q_weight_void);
const auto* k_weight = static_cast<const Weight*>(k_weight_void);
const auto* cos = static_cast<const Rope*>(cos_void);
const auto* sin = static_cast<const Rope*>(sin_void);
auto* k_cache = static_cast<Scalar*>(k_cache_void);
auto* v_cache = static_cast<Scalar*>(v_cache_void);
const int head_in_block = threadIdx.x / kThreadsPerHead;
const int lane = threadIdx.x % kThreadsPerHead;
const int total_heads = num_q_heads + 2 * num_kv_heads;
int token;
int local_head;
const int64_t global_head =
static_cast<int64_t>(blockIdx.x) * kHeadsPerBlock + head_in_block;
token = global_head / total_heads;
// Reuse the quotient instead of spelling this as global_head %
// total_heads. nvcc otherwise emits a second runtime integer-division
// sequence for the flattened token/head mapping.
local_head = static_cast<int>(global_head -
static_cast<int64_t>(token) * total_heads);
if (token >= num_tokens) return;
const bool is_q = local_head < num_q_heads;
const bool is_k = !is_q && local_head < num_q_heads + num_kv_heads;
const int head = is_q ? local_head
: (is_k ? local_head - num_q_heads
: local_head - num_q_heads - num_kv_heads);
int64_t input_offset = token * qkv_token_stride;
Scalar* output;
int64_t output_offset;
if (is_q) {
input_offset += static_cast<int64_t>(head) * HEAD_DIM;
output = q_out;
output_offset =
token * q_token_stride + static_cast<int64_t>(head) * HEAD_DIM;
} else if (is_k) {
input_offset += static_cast<int64_t>(num_q_heads + head) * HEAD_DIM;
output = k_out;
output_offset =
token * k_token_stride + static_cast<int64_t>(head) * HEAD_DIM;
} else {
input_offset +=
static_cast<int64_t>(num_q_heads + num_kv_heads + head) * HEAD_DIM;
output = v_out;
output_offset =
token * v_token_stride + static_cast<int64_t>(head) * HEAD_DIM;
}
bool write_cache = false;
int64_t cache_offset = 0;
if (!is_q && token < num_slots) {
const int64_t slot = slot_mapping[token];
if (slot >= 0) {
int64_t block;
int64_t page;
if (block_size_shift >= 0) {
// vLLM production cache blocks are powers of two. Avoid the costly
// runtime int64 division on that path while retaining generic block
// sizes through the fallback below.
block = slot >> block_size_shift;
page = slot & (block_size - 1);
} else {
block = slot / block_size;
page = slot - block * block_size;
}
cache_offset = block * cache_block_stride + page * cache_page_stride +
static_cast<int64_t>(head) * cache_head_stride;
write_cache = true;
}
}
if (!is_q && !is_k) {
#pragma unroll
for (int block = 0; block < kNumVectors; ++block) {
#pragma unroll
for (int element = 0; element < kValuesPerVector; ++element) {
const int dim = block * kThreadsPerHead * kValuesPerVector +
lane * kValuesPerVector + element;
if constexpr (kTileSize > HEAD_DIM) {
if (dim >= HEAD_DIM) continue;
}
const Scalar value = qkv[input_offset + dim];
output[output_offset + dim] = value;
if (write_cache) {
if constexpr (FP8_CACHE) {
reinterpret_cast<uint8_t*>(v_cache_void)[cache_offset + dim] =
fp8::scaled_convert<uint8_t, float,
Fp8KVCacheDataType::kFp8E4M3>(
ScalarConverter::convert(value), *v_scale);
} else {
v_cache[cache_offset + dim] = value;
}
}
}
}
return;
}
float values[kValuesPerThread];
float sum_squares = 0.0f;
#pragma unroll
for (int block = 0; block < kNumVectors; ++block) {
#pragma unroll
for (int element = 0; element < kValuesPerVector; ++element) {
const int value_index = block * kValuesPerVector + element;
const int dim = block * kThreadsPerHead * kValuesPerVector +
lane * kValuesPerVector + element;
float value = 0.0f;
if constexpr (kTileSize > HEAD_DIM) {
if (dim < HEAD_DIM) {
value = ScalarConverter::convert(qkv[input_offset + dim]);
}
} else {
value = ScalarConverter::convert(qkv[input_offset + dim]);
}
values[value_index] = value;
// CuTeDSL lowers its fragment reduction to a sequential fma.rn
// chain. Keep this explicit: changing it changes RMSNorm bits.
sum_squares = __fmaf_rn(value, value, sum_squares);
}
}
const int warp_lane = threadIdx.x % kWarpSize;
const int subwarp_base = (warp_lane / kThreadsPerHead) * kThreadsPerHead;
constexpr uint32_t kSubwarpBits =
kThreadsPerHead == kWarpSize ? 0xffffffffu
: ((uint32_t{1} << kThreadsPerHead) - 1);
const uint32_t subwarp_mask = kSubwarpBits << subwarp_base;
#pragma unroll
for (int offset = 1; offset < kThreadsPerHead; offset <<= 1) {
// CuTeDSL's warp_reduce uses ascending XOR offsets (1, 2, 4, ...).
sum_squares +=
__shfl_xor_sync(subwarp_mask, sum_squares, offset, kWarpSize);
}
const float mean_square =
__fdiv_rn(sum_squares, static_cast<float>(HEAD_DIM));
const float inverse_rms = rsqrtf(mean_square + epsilon);
const Weight* weight = is_q ? q_weight : k_weight;
// CuTeDSL writes RMSNorm results to activation-typed shared memory and
// reloads them for RoPE. This intermediate FP16/BF16 rounding boundary
// is intentional and is necessary for bitwise output parity.
__shared__ Scalar normalized[kHeadsPerBlock * HEAD_DIM];
Scalar* normalized_head = normalized + head_in_block * HEAD_DIM;
#pragma unroll
for (int block = 0; block < kNumVectors; ++block) {
#pragma unroll
for (int element = 0; element < kValuesPerVector; ++element) {
const int value_index = block * kValuesPerVector + element;
const int dim = block * kThreadsPerHead * kValuesPerVector +
lane * kValuesPerVector + element;
if constexpr (kTileSize > HEAD_DIM) {
if (dim >= HEAD_DIM) continue;
}
const float x_hat = values[value_index] * inverse_rms;
const float norm_weight =
WeightConverter::convert(weight[dim]) + norm_weight_bias;
normalized_head[dim] = ScalarConverter::convert(x_hat * norm_weight);
}
}
__syncwarp(subwarp_mask);
// Apply NeoX-style RoPE. CuTeDSL assigns four rotary pairs to each
// participating lane and reloads activation-typed normalized values.
constexpr int kRotaryPairsPerThread = 4;
if (lane < rotary_pairs / kRotaryPairsPerThread) {
const int64_t position = positions[token];
const Rope* token_cos = cos + position * cos_token_stride;
const Rope* token_sin = sin + position * sin_token_stride;
#pragma unroll
for (int element = 0; element < kRotaryPairsPerThread; ++element) {
const int pair = lane * kRotaryPairsPerThread + element;
const int dim0 = pair;
const int dim1 = pair + rotary_pairs;
const float value0 = ScalarConverter::convert(normalized_head[dim0]);
const float value1 = ScalarConverter::convert(normalized_head[dim1]);
const float cos_value = RopeConverter::convert(token_cos[pair]);
const float sin_value = RopeConverter::convert(token_sin[pair]);
// Match CuTeDSL PTX exactly. The first half is mul + mul + sub; the
// second is one mul followed by one fused multiply-add.
const float rotated0 = __fsub_rn(__fmul_rn(value0, cos_value),
__fmul_rn(value1, sin_value));
const float rotated1 =
__fmaf_rn(value0, sin_value, __fmul_rn(value1, cos_value));
normalized_head[dim0] = ScalarConverter::convert(rotated0);
normalized_head[dim1] = ScalarConverter::convert(rotated1);
}
}
__syncwarp(subwarp_mask);
#pragma unroll
for (int block = 0; block < kNumVectors; ++block) {
#pragma unroll
for (int element = 0; element < kValuesPerVector; ++element) {
const int dim = block * kThreadsPerHead * kValuesPerVector +
lane * kValuesPerVector + element;
if constexpr (kTileSize > HEAD_DIM) {
if (dim >= HEAD_DIM) continue;
}
const Scalar value = normalized_head[dim];
output[output_offset + dim] = value;
if (write_cache) {
if constexpr (FP8_CACHE) {
reinterpret_cast<uint8_t*>(k_cache_void)[cache_offset + dim] =
fp8::scaled_convert<uint8_t, float,
Fp8KVCacheDataType::kFp8E4M3>(
ScalarConverter::convert(value), *k_scale);
} else {
k_cache[cache_offset + dim] = value;
}
}
}
}
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800
}
#endif
}
template <typename scalar_t, typename weight_t, typename rope_t, bool FP8_CACHE>
void launch_qknorm_rope_cache(
const torch::stable::Tensor& qkv, torch::stable::Tensor& q_out,
torch::stable::Tensor& k_out, torch::stable::Tensor& v_out,
const torch::stable::Tensor& q_weight,
const torch::stable::Tensor& k_weight, const torch::stable::Tensor& cos,
const torch::stable::Tensor& sin, const torch::stable::Tensor& positions,
const torch::stable::Tensor& slot_mapping, torch::stable::Tensor& k_cache,
torch::stable::Tensor& v_cache, int head_dim, int num_q_heads,
int num_kv_heads, int rotary_pairs, float epsilon, float norm_weight_bias,
cudaStream_t stream, const float* k_scale, const float* v_scale) {
const int total_heads = num_q_heads + 2 * num_kv_heads;
const int64_t total_work = qkv.size(0) * total_heads;
const int block_size = static_cast<int>(k_cache.size(1));
const int block_size_shift =
(block_size & (block_size - 1)) == 0
? __builtin_ctz(static_cast<unsigned int>(block_size))
: -1;
#define LAUNCH_QKNORM_ROPE_CACHE(HEAD_DIM, WIDE_192_ROTARY) \
do { \
constexpr int threads_per_head = \
WIDE_192_ROTARY ? 32 \
: (HEAD_DIM <= 64 ? 8 : (HEAD_DIM <= 192 ? 16 : 32)); \
constexpr int heads_per_block = kThreadsPerBlock / threads_per_head; \
const dim3 grid((total_work + heads_per_block - 1) / heads_per_block); \
const dim3 block(kThreadsPerBlock); \
qknorm_rope_cache_kernel<scalar_t, weight_t, rope_t, HEAD_DIM, \
WIDE_192_ROTARY, FP8_CACHE> \
<<<grid, block, 0, stream>>>( \
qkv.data_ptr(), q_out.data_ptr(), k_out.data_ptr(), \
v_out.data_ptr(), q_weight.data_ptr(), k_weight.data_ptr(), \
cos.data_ptr(), sin.data_ptr(), \
reinterpret_cast<const int64_t*>(positions.data_ptr()), \
reinterpret_cast<const int64_t*>(slot_mapping.data_ptr()), \
k_cache.data_ptr(), v_cache.data_ptr(), qkv.stride(0), \
q_out.stride(0), k_out.stride(0), v_out.stride(0), cos.stride(0), \
sin.stride(0), k_cache.stride(0), k_cache.stride(1), \
k_cache.stride(2), static_cast<int>(qkv.size(0)), \
static_cast<int>(slot_mapping.numel()), num_q_heads, num_kv_heads, \
rotary_pairs, block_size, epsilon, block_size_shift, \
norm_weight_bias, k_scale, v_scale); \
} while (false)
switch (head_dim) {
case 64:
LAUNCH_QKNORM_ROPE_CACHE(64, false);
break;
case 128:
LAUNCH_QKNORM_ROPE_CACHE(128, false);
break;
case 192:
if (rotary_pairs > 64) {
LAUNCH_QKNORM_ROPE_CACHE(192, true);
} else {
LAUNCH_QKNORM_ROPE_CACHE(192, false);
}
break;
case 256:
LAUNCH_QKNORM_ROPE_CACHE(256, false);
break;
default:
STD_TORCH_CHECK(false, "Unsupported head_dim: ", head_dim);
}
#undef LAUNCH_QKNORM_ROPE_CACHE
}
void check_cuda_tensor(const torch::stable::Tensor& tensor, const char* name) {
STD_TORCH_CHECK(tensor.is_cuda(), name, " must be a CUDA tensor");
}
void check_same_device(const torch::stable::Tensor& reference,
const torch::stable::Tensor& tensor, const char* name) {
STD_TORCH_CHECK(tensor.get_device_index() == reference.get_device_index(),
name, " must be on the same CUDA device as qkv");
}
} // namespace
void fused_qknorm_rope_cache(
torch::stable::Tensor& q_out, torch::stable::Tensor& k_out,
torch::stable::Tensor& v_out, torch::stable::Tensor& qkv,
torch::stable::Tensor& q_weight, torch::stable::Tensor& k_weight,
torch::stable::Tensor& cos, torch::stable::Tensor& sin,
torch::stable::Tensor& positions, torch::stable::Tensor& slot_mapping,
torch::stable::Tensor& k_cache, torch::stable::Tensor& v_cache,
int64_t head_dim, int64_t num_q_heads, int64_t num_kv_heads,
int64_t rotary_pairs, double epsilon, double norm_weight_bias,
const std::string& kv_cache_dtype,
std::optional<torch::stable::Tensor> k_scale,
std::optional<torch::stable::Tensor> v_scale) {
check_cuda_tensor(qkv, "qkv");
check_cuda_tensor(q_out, "q_out");
check_cuda_tensor(k_out, "k_out");
check_cuda_tensor(v_out, "v_out");
check_cuda_tensor(q_weight, "q_weight");
check_cuda_tensor(k_weight, "k_weight");
check_cuda_tensor(cos, "cos");
check_cuda_tensor(sin, "sin");
check_cuda_tensor(positions, "positions");
check_cuda_tensor(slot_mapping, "slot_mapping");
check_cuda_tensor(k_cache, "k_cache");
check_cuda_tensor(v_cache, "v_cache");
check_same_device(qkv, q_out, "q_out");
check_same_device(qkv, k_out, "k_out");
check_same_device(qkv, v_out, "v_out");
check_same_device(qkv, q_weight, "q_weight");
check_same_device(qkv, k_weight, "k_weight");
check_same_device(qkv, cos, "cos");
check_same_device(qkv, sin, "sin");
check_same_device(qkv, positions, "positions");
check_same_device(qkv, slot_mapping, "slot_mapping");
check_same_device(qkv, k_cache, "k_cache");
check_same_device(qkv, v_cache, "v_cache");
STD_TORCH_CHECK(qkv.dim() == 2 && qkv.stride(1) == 1,
"qkv must be a 2D tensor contiguous in its last dimension");
STD_TORCH_CHECK(q_out.dim() == 2 && q_out.is_contiguous(),
"q_out must be contiguous and 2D");
STD_TORCH_CHECK(k_out.dim() == 2 && k_out.is_contiguous(),
"k_out must be contiguous and 2D");
STD_TORCH_CHECK(v_out.dim() == 2 && v_out.is_contiguous(),
"v_out must be contiguous and 2D");
STD_TORCH_CHECK(q_weight.dim() == 1 && q_weight.is_contiguous(),
"q_weight must be contiguous and 1D");
STD_TORCH_CHECK(k_weight.dim() == 1 && k_weight.is_contiguous(),
"k_weight must be contiguous and 1D");
STD_TORCH_CHECK(cos.dim() == 2 && cos.stride(1) == 1,
"cos must be 2D and contiguous in its last dimension");
STD_TORCH_CHECK(sin.dim() == 2 && sin.stride(1) == 1,
"sin must be 2D and contiguous in its last dimension");
STD_TORCH_CHECK(
positions.scalar_type() == torch::headeronly::ScalarType::Long,
"positions must be an int64 tensor");
STD_TORCH_CHECK(positions.is_contiguous(), "positions must be contiguous");
STD_TORCH_CHECK(
slot_mapping.scalar_type() == torch::headeronly::ScalarType::Long,
"slot_mapping must be an int64 tensor");
STD_TORCH_CHECK(slot_mapping.is_contiguous(),
"slot_mapping must be contiguous");
STD_TORCH_CHECK(k_cache.dim() == 4 && v_cache.dim() == 4,
"K/V cache must have shape [blocks, block_size, heads, dim]");
for (int64_t d = 0; d < 4; ++d) {
STD_TORCH_CHECK(k_cache.size(d) == v_cache.size(d) &&
k_cache.stride(d) == v_cache.stride(d),
"K/V cache must have identical shapes and strides");
}
STD_TORCH_CHECK(k_cache.stride(3) == 1,
"K/V cache head dimension must be contiguous");
STD_TORCH_CHECK(num_q_heads > 0 && num_kv_heads > 0,
"Q/KV head counts must be positive");
STD_TORCH_CHECK(
head_dim == 64 || head_dim == 128 || head_dim == 192 || head_dim == 256,
"Unsupported head_dim");
STD_TORCH_CHECK(cos.size(0) == sin.size(0),
"cos/sin cache must have the same number of positions");
const int64_t num_tokens = qkv.size(0);
const int64_t packed_width = (num_q_heads + 2 * num_kv_heads) * head_dim;
STD_TORCH_CHECK(qkv.size(1) == packed_width,
"qkv width must equal (num_q_heads + 2 * num_kv_heads) * "
"head_dim");
STD_TORCH_CHECK(
q_out.size(0) == num_tokens && q_out.size(1) == num_q_heads * head_dim,
"q_out has an invalid shape");
STD_TORCH_CHECK(
k_out.size(0) == num_tokens && k_out.size(1) == num_kv_heads * head_dim,
"k_out has an invalid shape");
STD_TORCH_CHECK(
v_out.size(0) == k_out.size(0) && v_out.size(1) == k_out.size(1),
"v_out has an invalid shape");
STD_TORCH_CHECK(q_weight.numel() == head_dim && k_weight.numel() == head_dim,
"Q/K norm weights must have head_dim elements");
STD_TORCH_CHECK(positions.numel() == num_tokens,
"positions must have one entry per qkv token");
STD_TORCH_CHECK(rotary_pairs >= 0 && rotary_pairs * 2 <= head_dim &&
rotary_pairs % 4 == 0,
"rotary_pairs must be a multiple of 4 in [0, head_dim / 2]");
STD_TORCH_CHECK(cos.size(1) >= rotary_pairs && sin.size(1) >= rotary_pairs,
"cos/sin cache is smaller than rotary_pairs");
STD_TORCH_CHECK(
k_cache.size(2) == num_kv_heads && k_cache.size(3) == head_dim,
"K/V cache head shape does not match the kernel arguments");
STD_TORCH_CHECK(k_cache.size(1) > 0, "K/V cache block_size must be positive");
STD_TORCH_CHECK(slot_mapping.numel() <= num_tokens,
"slot_mapping cannot contain more entries than qkv tokens");
STD_TORCH_CHECK(qkv.scalar_type() == q_out.scalar_type() &&
qkv.scalar_type() == k_out.scalar_type() &&
qkv.scalar_type() == v_out.scalar_type(),
"qkv and Q/K/V outputs must have the same dtype");
const bool fp8_cache =
kv_cache_dtype == "fp8" || kv_cache_dtype == "fp8_e4m3";
STD_TORCH_CHECK(fp8_cache || kv_cache_dtype == "auto" ||
kv_cache_dtype == "bfloat16" ||
kv_cache_dtype == "float16",
"Unsupported KV cache dtype: ", kv_cache_dtype);
STD_TORCH_CHECK(k_cache.scalar_type() == v_cache.scalar_type(),
"K/V caches must have the same dtype");
if (fp8_cache) {
STD_TORCH_CHECK(
k_cache.scalar_type() == torch::headeronly::ScalarType::Byte ||
k_cache.scalar_type() ==
torch::headeronly::ScalarType::Float8_e4m3fn,
"FP8 cache requires uint8 or float8_e4m3fn storage");
STD_TORCH_CHECK(k_scale.has_value() && v_scale.has_value(),
"FP8 cache requires K/V scales");
for (const auto* scale : {&*k_scale, &*v_scale}) {
check_cuda_tensor(*scale, "KV scale");
check_same_device(qkv, *scale, "KV scale");
STD_TORCH_CHECK(
scale->numel() == 1 &&
scale->scalar_type() == torch::headeronly::ScalarType::Float,
"KV scales must be single FP32 values");
}
} else {
STD_TORCH_CHECK(qkv.scalar_type() == k_cache.scalar_type(),
"Unquantized cache must match QKV dtype");
}
STD_TORCH_CHECK(q_weight.scalar_type() == k_weight.scalar_type(),
"Q/K norm weights must have the same dtype");
STD_TORCH_CHECK(cos.scalar_type() == sin.scalar_type(),
"cos and sin must have the same dtype");
if (num_tokens == 0) return;
const torch::stable::accelerator::DeviceGuard device_guard(
qkv.get_device_index());
const cudaStream_t stream = get_current_cuda_stream(qkv.get_device_index());
VLLM_STABLE_DISPATCH_HALF_TYPES(
qkv.scalar_type(), "qknorm_rope_cache_input", [&] {
using input_t = scalar_t;
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
q_weight.scalar_type(), "qknorm_rope_cache_weight", [&] {
using norm_weight_t = scalar_t;
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
cos.scalar_type(), "qknorm_rope_cache_rope", [&] {
using rope_cache_t = scalar_t;
auto launch = [&](auto fp8_tag) {
launch_qknorm_rope_cache<input_t, norm_weight_t,
rope_cache_t,
decltype(fp8_tag)::value>(
qkv, q_out, k_out, v_out, q_weight, k_weight, cos,
sin, positions, slot_mapping, k_cache, v_cache,
static_cast<int>(head_dim),
static_cast<int>(num_q_heads),
static_cast<int>(num_kv_heads),
static_cast<int>(rotary_pairs),
static_cast<float>(epsilon),
static_cast<float>(norm_weight_bias), stream,
fp8_cache
? static_cast<const float*>(k_scale->data_ptr())
: nullptr,
fp8_cache
? static_cast<const float*>(v_scale->data_ptr())
: nullptr);
};
if (fp8_cache) {
launch(std::true_type{});
} else {
launch(std::false_type{});
}
});
});
});
STD_CUDA_KERNEL_LAUNCH_CHECK();
}
} // namespace vllm
+100
View File
@@ -0,0 +1,100 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#include <cuda_runtime.h>
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include <algorithm>
#include "dispatch_utils.h"
#include "quantization/vectorization_utils.cuh"
#include "torch_utils.h"
namespace vllm {
// 16 B of output per thread; low-precision inputs are widened in registers.
static constexpr int kMuseSoftCapVecSize = 4;
// blockIdx.x walks rows, blockIdx.y splits a row. Rows are the sampled tokens
// (often one, at decode) while a row is the whole vocab, so splitting the row
// is what keeps the GPU busy; a row-per-block grid would leave a single-token
// decode on one SM.
template <typename scalar_t>
__global__ void muse_spark_mul_tanh_soft_cap_kernel(
float* __restrict__ out, const scalar_t* __restrict__ input, int d,
float multiplier, float soft_cap_temp) {
const int64_t row_offset = static_cast<int64_t>(blockIdx.x) * d;
const scalar_t* __restrict__ in_row = input + row_offset;
float* __restrict__ out_row = out + row_offset;
const int tid = blockIdx.y * blockDim.x + threadIdx.x;
const int stride = gridDim.y * blockDim.x;
vectorize_with_alignment<kMuseSoftCapVecSize>(
in_row, out_row, d, tid, stride,
[=] __device__(float& dst, const scalar_t& src) {
// Multiply then divide, as MulTanhSoftCap.forward does, so fp32
// rounding matches the eager path.
dst = soft_cap_temp *
tanhf(static_cast<float>(src) * multiplier / soft_cap_temp);
});
}
} // namespace vllm
// out = soft_cap_temp * tanh(input * multiplier / soft_cap_temp), fp32.
void muse_spark_mul_tanh_soft_cap(torch::stable::Tensor& out,
torch::stable::Tensor& input,
double multiplier, double soft_cap_temp) {
STD_TORCH_CHECK(
input.scalar_type() == torch::headeronly::ScalarType::Float ||
input.scalar_type() == torch::headeronly::ScalarType::Half ||
input.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"muse_spark_mul_tanh_soft_cap: input must be float32, float16 or "
"bfloat16");
STD_TORCH_CHECK(out.scalar_type() == torch::headeronly::ScalarType::Float,
"muse_spark_mul_tanh_soft_cap: out must be float32");
STD_TORCH_CHECK(input.is_contiguous(),
"muse_spark_mul_tanh_soft_cap: input must be contiguous");
STD_TORCH_CHECK(out.is_contiguous(),
"muse_spark_mul_tanh_soft_cap: out must be contiguous");
STD_TORCH_CHECK(out.numel() == input.numel(),
"muse_spark_mul_tanh_soft_cap: out and input must have the "
"same number of elements");
STD_TORCH_CHECK(
soft_cap_temp != 0.0,
"muse_spark_mul_tanh_soft_cap: soft_cap_temp must be nonzero");
if (input.numel() == 0) {
return;
}
const int d = static_cast<int>(input.size(input.dim() - 1));
const int64_t num_tokens = input.numel() / d;
constexpr int kThreads = 128;
const int64_t vecs_per_row =
(d + vllm::kMuseSoftCapVecSize - 1) / vllm::kMuseSoftCapVecSize;
const int64_t blocks_to_cover = (vecs_per_row + kThreads - 1) / kThreads;
constexpr int64_t kTargetBlocks = 32768;
const int64_t blocks_per_row = std::clamp<int64_t>(
kTargetBlocks / num_tokens, 1, std::max<int64_t>(blocks_to_cover, 1));
dim3 grid(static_cast<unsigned>(num_tokens),
static_cast<unsigned>(blocks_per_row));
dim3 block(kThreads);
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "muse_spark_mul_tanh_soft_cap", [&] {
vllm::muse_spark_mul_tanh_soft_cap_kernel<scalar_t>
<<<grid, block, 0, stream>>>(out.mutable_data_ptr<float>(),
input.const_data_ptr<scalar_t>(), d,
static_cast<float>(multiplier),
static_cast<float>(soft_cap_temp));
});
}
+33
View File
@@ -363,6 +363,35 @@ minimax_allreduce_rms_qk(torch::stable::Tensor qkv,
int64_t const nranks, double const eps);
#endif
#ifndef USE_ROCM
namespace vllm {
void fused_qknorm_rope_cache(
torch::stable::Tensor& q_out, torch::stable::Tensor& k_out,
torch::stable::Tensor& v_out, torch::stable::Tensor& qkv,
torch::stable::Tensor& q_weight, torch::stable::Tensor& k_weight,
torch::stable::Tensor& cos, torch::stable::Tensor& sin,
torch::stable::Tensor& positions, torch::stable::Tensor& slot_mapping,
torch::stable::Tensor& k_cache, torch::stable::Tensor& v_cache,
int64_t head_dim, int64_t num_q_heads, int64_t num_kv_heads,
int64_t rotary_pairs, double epsilon, double norm_weight_bias,
const std::string& kv_cache_dtype,
std::optional<torch::stable::Tensor> k_scale,
std::optional<torch::stable::Tensor> v_scale);
} // namespace vllm
#endif
void fused_muse_spark_qknorm_rope_kv_insert(
torch::stable::Tensor& qkv, torch::stable::Tensor const& positions,
int64_t num_heads, int64_t num_kv_heads, int64_t head_dim, double eps,
std::optional<torch::stable::Tensor> cos_sin_cache, int64_t rotary_dim,
std::optional<torch::stable::Tensor> slot_mapping,
std::optional<torch::stable::Tensor> kv_cache, int64_t block_size,
const std::string& kv_cache_dtype,
std::optional<torch::stable::Tensor> k_scale,
std::optional<torch::stable::Tensor> v_scale,
std::optional<torch::stable::Tensor> q_fp8_out,
std::optional<torch::stable::Tensor> q_scale);
// Horizontally-fused MiniMax-M3 QK-norm + partial NeoX RoPE (+ optional KV /
// index-cache insert). Dense layer: norm+RoPE only; sparse layer: also packs
// the index branch and scatters k/v/index_k into their paged caches.
@@ -546,6 +575,10 @@ std::tuple<int64_t, torch::stable::Tensor> allocate_shared_buffer_and_handle(
int64_t open_mem_handle(torch::stable::Tensor& mem_handle);
void free_shared_buffer(int64_t buffer);
void muse_spark_mul_tanh_soft_cap(torch::stable::Tensor& out,
torch::stable::Tensor& input,
double multiplier, double soft_cap_temp);
// Activation kernels (shared CUDA/ROCm)
void silu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input);
void silu_and_mul_clamp(torch::stable::Tensor& out,
+31
View File
@@ -501,6 +501,27 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"float eps) -> (Tensor, Tensor)");
#endif
#ifndef USE_ROCM
ops.def(
"fused_qknorm_rope_cache("
"Tensor! q_out, Tensor! k_out, Tensor! v_out, Tensor qkv, "
"Tensor q_weight, Tensor k_weight, Tensor cos, Tensor sin, "
"Tensor positions, Tensor slot_mapping, Tensor! k_cache, "
"Tensor! v_cache, int head_dim, int num_q_heads, int num_kv_heads, "
"int rotary_pairs, float epsilon, float norm_weight_bias, "
"str kv_cache_dtype=\"auto\", Tensor? k_scale=None, Tensor? "
"v_scale=None) -> ()");
#endif
ops.def(
"fused_muse_spark_qknorm_rope_kv_insert("
"Tensor! qkv, Tensor positions, int num_heads, int num_kv_heads, "
"int head_dim, float eps, Tensor? cos_sin_cache, int rotary_dim, "
"Tensor? slot_mapping, Tensor!? kv_cache, int block_size, "
"str kv_cache_dtype, Tensor? k_scale=None, Tensor? v_scale=None, "
"Tensor!? q_fp8_out=None, Tensor? q_scale=None) "
"-> ()");
// Horizontally-fused MiniMax-M3 QK-norm + partial NeoX RoPE + KV-insert.
ops.def(
"fused_minimax_m3_qknorm_rope_kv_insert("
@@ -598,6 +619,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"y_q, Tensor! y_s, bool use_ue8m0) -> ()");
ops.def("weak_ref_tensor(Tensor input) -> Tensor");
ops.def(
"muse_spark_mul_tanh_soft_cap(Tensor! out, Tensor input, "
"float multiplier, float soft_cap_temp) -> ()");
// Activation function used in SwiGLU.
ops.def("silu_and_mul(Tensor! result, Tensor input) -> ()");
@@ -828,7 +853,11 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
TORCH_BOX(&fused_kimi_k3_mla_decode_q_concat_ds_mla_insert));
#ifndef USE_ROCM
ops.impl("minimax_allreduce_rms_qk", TORCH_BOX(&minimax_allreduce_rms_qk));
ops.impl("fused_qknorm_rope_cache",
TORCH_BOX(&vllm::fused_qknorm_rope_cache));
#endif
ops.impl("fused_muse_spark_qknorm_rope_kv_insert",
TORCH_BOX(&fused_muse_spark_qknorm_rope_kv_insert));
ops.impl("fused_minimax_m3_qknorm_rope_kv_insert",
TORCH_BOX(&fused_minimax_m3_qknorm_rope_kv_insert));
#ifdef VLLM_ENABLE_FUSED_KDA_DECODE
@@ -864,6 +893,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
TORCH_BOX(&persistent_masked_m_silu_mul_quant));
ops.impl("weak_ref_tensor", TORCH_BOX(&weak_ref_tensor));
ops.impl("silu_and_mul_quant", TORCH_BOX(&silu_and_mul_quant));
ops.impl("muse_spark_mul_tanh_soft_cap",
TORCH_BOX(&muse_spark_mul_tanh_soft_cap));
ops.impl("silu_and_mul", TORCH_BOX(&silu_and_mul));
ops.impl("mul_and_silu", TORCH_BOX(&mul_and_silu));
ops.impl("gelu_and_mul", TORCH_BOX(&gelu_and_mul));