[ROCm][MiniMax-M3] Add AITER sparse paged attention (#47287)

Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: Jun Kang Chow <junkangchow@gmail.com>
Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
Tan Pin Siang
2026-07-12 19:27:29 -07:00
committed by GitHub
co-authored by vllmellm Hongxia Yang Jun Kang Chow tjtanaa
parent e26264f3ef
commit ee5a89f4d7
15 changed files with 1554 additions and 128 deletions
@@ -39,12 +39,10 @@
* The IQ/IK warps address the index_q/index_k sub-blocks *inside* qkv at the
* fixed physical offsets (nq+2*nkv)*128 and (nq+2*nkv+niq)*128.
*
* Dense vs sparse is a compile-time choice via the ``kIsSparse``/``kInsertKV``
* template bools (3 instantiations: dense <false,false>, sparse-profiling
* <true,false>, sparse-serving <true,true>), so the index slots, the V slots
* and the cache inserts fold away entirely on paths that don't use them. The
* dense layer passes no caches/index: norm+RoPE happens in place and the
* generic ``Attention`` layer owns the cache write.
* Dense vs sparse row layout and index-branch processing are separate template
* choices. Skip-index-topk reuse layers still have sparse rows and insert main
* K/V cache entries, but compile away index_q/index_k work and index-cache
* writes.
*
* Q/K and (sparse) index_q/index_k are all rewritten in place inside the fused
* ``qkv`` tensor. Caches (bf16) are scatter-written by slot.
@@ -223,10 +221,25 @@ __device__ __forceinline__ void storeCacheElems(
// model dtype directly. FP8 cache dtypes use the conversion path below.
storeElems<scalar_t>(reinterpret_cast<scalar_t*>(dst), elems);
} else {
#pragma unroll
#ifdef USE_ROCM
// Match ROCm's model-dtype materialization before FP8 cache conversion.
using Converter = vllm::_typeConvert<scalar_t>;
using rounded_t = typename Converter::hip_type;
rounded_t rounded[kElemsPerLane];
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
rounded[i] = Converter::convert(elems[i]);
}
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
dst[i] = fp8::scaled_convert<cache_t, rounded_t, kv_dt>(rounded[i], 1.0f);
}
#else
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
dst[i] = fp8::scaled_convert<cache_t, float, kv_dt>(elems[i], 1.0f);
}
#endif
}
}
@@ -262,20 +275,25 @@ __device__ __forceinline__ void storeElemsFp8(
// Grid: 1D, ceil(num_tokens * slots_per_token / warps_per_block).
// Each warp = one (token, slot).
//
// `kIsSparse` and `kInsertKV` are compile-time template bools, so all the
// branch decisions that distinguish the dense layer from the sparse layer
// (index slots, KV/index inserts, V slots) fold away per instantiation.
// Three instantiations are built: dense <false,false>, sparse-profiling
// <true,false> and sparse-serving <true,true>. Slots per token:
// `kHasIndex`, `kProcessIndex`, and `kInsertKV` are compile-time template
// bools, so branch decisions that distinguish the dense layer from the sparse
// layer (index slots, KV/index inserts, V slots) fold away per instantiation.
// Slots per token:
// Q : nq (always — norm+RoPE)
// K : nkv (always — norm+RoPE; +K-cache insert)
// V : nkv only if kInsertKV (V-cache insert; no warps in dense)
// IQ: niq only if kIsSparse (norm+RoPE)
// IK: 1 only if kIsSparse (norm+RoPE; +index-cache insert)
// IQ: niq only if kProcessIndex (norm+RoPE)
// IK: 1 only if kProcessIndex (norm+RoPE; +index-cache insert)
// cache_t/kv_dt: main attention KV-cache dtype (auto/fp8). out_idx_t/kFp8Idx:
// indexer index-K cache + index-Q output dtype (scalar_t or e4m3 byte).
// kHasIndex means the qkv row is laid out as sparse [q|k|v|index_q|index_k].
// kProcessIndex controls whether this launch actually norms/ropes the index
// branch and writes index_q/index_k outputs. Skip-index-topk reuse layers keep
// kHasIndex=true but set kProcessIndex=false.
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt,
typename out_idx_t, bool kIsSparse, bool kInsertKV, bool kFp8Idx>
typename out_idx_t, bool kHasIndex, bool kInsertKV,
bool kProcessIndex,
bool kFp8Idx>
__global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
scalar_t* __restrict__ qkv, // [N, qkv_row] in/out (packs index if sparse)
scalar_t* __restrict__ q_out, // [N, nq*128] contiguous, or nullptr
@@ -308,9 +326,12 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
int const laneId = threadIdx.x % 32;
int const globalWarpIdx = blockIdx.x * warpsPerBlock + (threadIdx.x / 32);
static_assert(!kProcessIndex || kHasIndex,
"index processing requires sparse row layout");
// Slot layout (compile-time gated: dense has neither V nor index slots).
int const v_slots = kInsertKV ? nkv : 0;
int const idx_slots = kIsSparse ? niq + 1 : 0;
int const idx_slots = kProcessIndex ? niq + 1 : 0;
int const slots_per_token = nq + nkv + v_slots + idx_slots;
int const tokenIdx = globalWarpIdx / slots_per_token;
@@ -321,14 +342,14 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
int const k_begin = nq;
int const v_begin = nq + nkv; // valid only when kInsertKV
int const iq_begin = nq + nkv + v_slots; // index block start
int const ik_slot = iq_begin + niq; // valid only when kIsSparse
int const ik_slot = iq_begin + niq; // valid only when kProcessIndex
bool const isQ = slot < k_begin;
bool const isK = slot >= k_begin && slot < v_begin;
bool isV = false;
if constexpr (kInsertKV) isV = slot >= v_begin && slot < v_begin + nkv;
bool isIQ = false, isIK = false;
if constexpr (kIsSparse) {
if constexpr (kProcessIndex) {
isIQ = slot >= iq_begin && slot < ik_slot;
isIK = slot == ik_slot;
}
@@ -336,7 +357,7 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
int const dim_base = laneId * kElemsPerLane;
// Physical row width of qkv: the dense layer packs [q|k|v]; the sparse
// layer additionally packs [index_q (niq heads) | index_k (1 head)].
int const qkv_row = (nq + 2 * nkv + (kIsSparse ? (niq + 1) : 0)) * kHeadDim;
int const qkv_row = (nq + 2 * nkv + (kHasIndex ? (niq + 1) : 0)) * kHeadDim;
// ── Resolve source pointer + per-branch parameters. ────────────────────
scalar_t* row_ptr = nullptr; // in-place output location
@@ -367,10 +388,13 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
row_ptr = qkv + static_cast<int64_t>(tokenIdx) * qkv_row +
(nq + 2 * nkv + ih) * kHeadDim;
norm_w = iq_norm_w;
} else { // isIK -- single shared index key at (nq+2*nkv+niq)*128.
} else if (isIK) {
// Single shared index key at (nq+2*nkv+niq)*128.
row_ptr = qkv + static_cast<int64_t>(tokenIdx) * qkv_row +
(nq + 2 * nkv + niq) * kHeadDim;
norm_w = ik_norm_w;
} else {
return;
}
// Store destination. Q and index_q are gathered into dedicated contiguous
@@ -426,9 +450,12 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
// ── Cache inserts (sparse serving only). ───────────────────────────────
if constexpr (kInsertKV) {
// Guard (not early-return) so every thread reaches the PDL trigger below.
int64_t const sm = (isK || isV)
? slot_mapping[tokenIdx]
: (isIK ? index_slot_mapping[tokenIdx] : -1);
int64_t sm = -1;
if (isK || isV) {
sm = slot_mapping[tokenIdx];
} else if constexpr (kProcessIndex) {
if (isIK) sm = index_slot_mapping[tokenIdx];
}
if (sm >= 0) { // skip padded / unscheduled tokens
if (isIK) {
if constexpr (kFp8Idx) {
@@ -475,12 +502,12 @@ void launchFusedMiniMaxM3(
int const nkv, int const niq, int const block_size,
int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token,
int64_t const kv_s_dim, bool const has_index, bool const insert_kv,
bool const fp8_idx, cudaStream_t stream) {
bool const process_index, bool const fp8_idx, cudaStream_t stream) {
// Index outputs are scalar_t (bf16) or e4m3 bytes (uint8_t); reinterpret the
// void* pointers per instantiation in the LAUNCH macro.
// Slot count must match the kernel's compile-time gating.
int const v_slots = insert_kv ? nkv : 0;
int const idx_slots = has_index ? niq + 1 : 0;
int const idx_slots = process_index ? niq + 1 : 0;
int const slots_per_token = nq + nkv + v_slots + idx_slots;
constexpr int kBlockSize = 256;
@@ -507,11 +534,12 @@ void launchFusedMiniMaxM3(
config.attrs = attrs;
config.numAttrs = (sm_version >= 90) ? 1 : 0;
#define LAUNCH(IS_SPARSE, INSERT, FP8, OUT_T) \
#define LAUNCH(HAS_INDEX, INSERT, PROCESS_INDEX, FP8, OUT_T) \
cudaLaunchKernelEx( \
&config, \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, cache_t, kv_dt, OUT_T, \
IS_SPARSE, INSERT, FP8>, \
HAS_INDEX, INSERT, \
PROCESS_INDEX, FP8>, \
qkv, q_out, reinterpret_cast<OUT_T*>(index_q_out), q_norm_w, k_norm_w, \
iq_norm_w, ik_norm_w, cos_sin_cache, positions, slot_mapping, \
index_slot_mapping, kv_cache, reinterpret_cast<OUT_T*>(index_cache), \
@@ -520,37 +548,45 @@ void launchFusedMiniMaxM3(
#else
// ROCm: standard kernel launch syntax (no PDL/stream serialization).
// clang-format off
#define LAUNCH(IS_SPARSE, INSERT, FP8, OUT_T) \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, cache_t, kv_dt, OUT_T, \
IS_SPARSE, INSERT, FP8> \
<<<grid, kBlockSize, 0, stream>>>( \
qkv, q_out, reinterpret_cast<OUT_T*>(index_q_out), q_norm_w, \
k_norm_w, iq_norm_w, ik_norm_w, cos_sin_cache, positions, \
slot_mapping, index_slot_mapping, kv_cache, \
reinterpret_cast<OUT_T*>(index_cache), eps, rotary_dim, \
num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_head, \
kv_s_token, kv_s_dim)
#define LAUNCH(HAS_INDEX, INSERT, PROCESS_INDEX, FP8, OUT_T) \
fusedMiniMaxM3QNormRopeKVInsertKernel< \
scalar_t, cache_t, kv_dt, OUT_T, HAS_INDEX, INSERT, PROCESS_INDEX, \
FP8><<<grid, kBlockSize, 0, stream>>>( \
qkv, q_out, reinterpret_cast<OUT_T*>(index_q_out), q_norm_w, \
k_norm_w, iq_norm_w, ik_norm_w, cos_sin_cache, positions, \
slot_mapping, index_slot_mapping, kv_cache, \
reinterpret_cast<OUT_T*>(index_cache), eps, rotary_dim, num_tokens, \
nq, nkv, niq, block_size, kv_s_block, kv_s_head, kv_s_token, \
kv_s_dim)
// clang-format on
#endif
if (has_index) {
if (insert_kv) {
if (fp8_idx) {
LAUNCH(true, true, true, uint8_t); // sparse serving, fp8 index outputs
if (!process_index) {
if (insert_kv) {
LAUNCH(true, true, false, false, scalar_t);
} else {
LAUNCH(true, true, false, scalar_t); // sparse serving, bf16
LAUNCH(true, false, false, false, scalar_t);
}
} else if (insert_kv) {
if (fp8_idx) {
LAUNCH(true, true, true, true,
uint8_t); // sparse serving, fp8 index outputs
} else {
LAUNCH(true, true, true, false, scalar_t); // sparse serving, bf16
}
} else {
if (fp8_idx) {
LAUNCH(true, false, true, uint8_t); // sparse profiling, fp8 index_q
LAUNCH(true, false, true, true,
uint8_t); // sparse profiling, fp8 index_q
} else {
LAUNCH(true, false, false, scalar_t); // sparse profiling, bf16
LAUNCH(true, false, true, false, scalar_t); // sparse profiling, bf16
}
}
} else {
// Dense layer: never has an index branch and never inserts here (the
// generic Attention layer owns the KV insert).
LAUNCH(false, false, false, scalar_t);
LAUNCH(false, false, false, false, scalar_t);
}
#undef LAUNCH
}
@@ -558,6 +594,7 @@ void launchFusedMiniMaxM3(
} // namespace minimax_m3_fused_ops
} // namespace vllm
// clang-format off
#define CALL_FUSED_MINIMAX_M3(_RAW_T, CACHE_T, KV_DTYPE) \
vllm::minimax_m3_fused_ops::launchFusedMiniMaxM3<st, CACHE_T, KV_DTYPE>( \
reinterpret_cast<st*>(qkv.data_ptr()), \
@@ -567,24 +604,29 @@ void launchFusedMiniMaxM3(
: nullptr, \
reinterpret_cast<st const*>(q_norm_weight.data_ptr()), \
reinterpret_cast<st const*>(k_norm_weight.data_ptr()), \
has_index ? reinterpret_cast<st const*>(index_q_norm_weight->data_ptr()) \
: nullptr, \
has_index ? reinterpret_cast<st const*>(index_k_norm_weight->data_ptr()) \
: nullptr, \
process_index \
? reinterpret_cast<st const*>(index_q_norm_weight->data_ptr()) \
: nullptr, \
process_index \
? reinterpret_cast<st const*>(index_k_norm_weight->data_ptr()) \
: nullptr, \
reinterpret_cast<st const*>(cos_sin_cache.data_ptr()), \
reinterpret_cast<int64_t const*>(positions.data_ptr()), \
insert_kv ? reinterpret_cast<int64_t const*>(slot_mapping->data_ptr()) \
: nullptr, \
insert_kv ? reinterpret_cast<int64_t const*>( \
effective_index_slot_mapping->data_ptr()) \
: nullptr, \
(insert_kv && process_index) \
? reinterpret_cast<int64_t const*>( \
effective_index_slot_mapping->data_ptr()) \
: nullptr, \
insert_kv ? reinterpret_cast<CACHE_T*>(kv_cache->data_ptr()) : nullptr, \
(insert_kv && has_index) \
(insert_kv && process_index) \
? reinterpret_cast<void*>(index_cache->data_ptr()) \
: nullptr, \
static_cast<float>(eps), static_cast<int>(rotary_dim), num_tokens, nq, \
nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_head, \
kv_s_token, kv_s_dim, has_index, insert_kv, fp8_idx, stream)
kv_s_token, kv_s_dim, has_index, insert_kv, process_index, fp8_idx, \
stream)
// clang-format on
// ────────────────────────────────────────────────────────────────────────────
// Torch op wrapper
@@ -607,7 +649,7 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
std::optional<torch::stable::Tensor> q_out, // [N, nq*128] contiguous
std::optional<torch::stable::Tensor>
index_q_out, // [N, niq*128] contiguous
const std::string& kv_cache_dtype) {
const std::string& kv_cache_dtype, bool skip_index_branch) {
STD_TORCH_CHECK(qkv.is_cuda() && qkv.is_contiguous(),
"qkv must be contiguous CUDA");
STD_TORCH_CHECK(
@@ -646,6 +688,7 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
// (1 head)]) right after [q|k|v] in the same row; the dense layer does not.
bool const has_index = niq > 0;
bool const insert_kv = kv_cache.has_value();
bool const process_index = has_index && !skip_index_branch;
vllm::Fp8KVCacheDataType const kv_dt =
vllm::get_fp8_kv_cache_data_type(kv_cache_dtype);
int const kHeadDim = vllm::minimax_m3_fused_ops::kHeadDim;
@@ -661,7 +704,9 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
STD_TORCH_CHECK(
!insert_kv || has_index,
"insert mode (kv_cache) requires the index branch (sparse layer)");
if (has_index) {
STD_TORCH_CHECK(has_index || !skip_index_branch,
"skip_index_branch requires sparse qkv rows");
if (process_index) {
STD_TORCH_CHECK(
index_q_norm_weight.has_value() && index_k_norm_weight.has_value(),
"index branch requires both index norm weights");
@@ -683,13 +728,15 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
slot_mapping.has_value() && slot_mapping->is_cuda() &&
slot_mapping->scalar_type() == torch::headeronly::ScalarType::Long,
"insert mode requires int64 CUDA slot_mapping");
STD_TORCH_CHECK(
!index_slot_mapping.has_value() ||
(index_slot_mapping->is_cuda() &&
index_slot_mapping->scalar_type() ==
torch::headeronly::ScalarType::Long &&
index_slot_mapping->numel() == slot_mapping->numel()),
"index_slot_mapping must be int64 CUDA with slot_mapping length");
if (process_index) {
STD_TORCH_CHECK(
!index_slot_mapping.has_value() ||
(index_slot_mapping->is_cuda() &&
index_slot_mapping->scalar_type() ==
torch::headeronly::ScalarType::Long &&
index_slot_mapping->numel() == slot_mapping->numel()),
"index_slot_mapping must be int64 CUDA with slot_mapping length");
}
// Main attention KV cache: auto matches qkv, fp8 uses uint8 storage.
if (kv_dt == vllm::Fp8KVCacheDataType::kAuto) {
STD_TORCH_CHECK(kv_cache->scalar_type() == qkv.scalar_type(),
@@ -700,12 +747,14 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
"fp8 kv_cache must use uint8 storage");
}
// Indexer index-K cache: independent dtype -- qkv dtype or fp8 e4m3.
STD_TORCH_CHECK(
index_cache.has_value() &&
(index_cache->scalar_type() == qkv.scalar_type() ||
index_cache->scalar_type() ==
torch::headeronly::ScalarType::Float8_e4m3fn),
"insert mode requires index_cache matching qkv dtype or fp8 e4m3");
if (process_index) {
STD_TORCH_CHECK(
index_cache.has_value() &&
(index_cache->scalar_type() == qkv.scalar_type() ||
index_cache->scalar_type() ==
torch::headeronly::ScalarType::Float8_e4m3fn),
"insert mode requires index_cache matching qkv dtype or fp8 e4m3");
}
STD_TORCH_CHECK(kv_cache->dim() == 4 && kv_cache->stride(3) == 1,
"kv_cache must be [nb,nkv,bs,2*head_dim] with contiguous "
"content dim (stride(3)==1)");
@@ -713,9 +762,11 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
kv_s_head = kv_cache->stride(1);
kv_s_token = kv_cache->stride(2);
kv_s_dim = kv_cache->stride(3);
effective_index_slot_mapping = index_slot_mapping.has_value()
? &index_slot_mapping.value()
: &slot_mapping.value();
if (process_index) {
effective_index_slot_mapping = index_slot_mapping.has_value()
? &index_slot_mapping.value()
: &slot_mapping.value();
}
}
// Optional contiguous gather targets: when given, the normed/roped q (and
// index_q) are written here instead of in place, so callers avoid a separate
@@ -730,9 +781,8 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
"q_out must have num_tokens * num_heads * 128 elements");
}
if (index_q_out.has_value()) {
STD_TORCH_CHECK(
has_index,
"index_q_out requires the index branch (num_index_heads > 0)");
STD_TORCH_CHECK(process_index,
"index_q_out requires index branch processing");
STD_TORCH_CHECK(
index_q_out->is_cuda() && index_q_out->is_contiguous() &&
(index_q_out->scalar_type() == qkv.scalar_type() ||
@@ -749,8 +799,9 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
// q/k/v + q_out stay qkv dtype. Both index outputs must agree.
auto const kFp8 = torch::headeronly::ScalarType::Float8_e4m3fn;
bool const fp8_idx =
(index_cache.has_value() && index_cache->scalar_type() == kFp8) ||
(index_q_out.has_value() && index_q_out->scalar_type() == kFp8);
process_index &&
((index_cache.has_value() && index_cache->scalar_type() == kFp8) ||
(index_q_out.has_value() && index_q_out->scalar_type() == kFp8));
if (fp8_idx) {
STD_TORCH_CHECK(
!index_cache.has_value() || index_cache->scalar_type() == kFp8,
+1 -1
View File
@@ -313,7 +313,7 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
std::optional<torch::stable::Tensor> index_cache, int64_t block_size,
std::optional<torch::stable::Tensor> q_out,
std::optional<torch::stable::Tensor> index_q_out,
const std::string& kv_cache_dtype);
const std::string& kv_cache_dtype, bool skip_index_branch);
// Sampler kernels (shared CUDA/ROCm)
void apply_repetition_penalties_(
+1 -1
View File
@@ -466,7 +466,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"Tensor? slot_mapping, Tensor? index_slot_mapping, "
"Tensor!? kv_cache, Tensor!? index_cache, "
"int block_size, Tensor!? q_out, Tensor!? index_q_out, "
"str kv_cache_dtype) -> ()");
"str kv_cache_dtype, bool skip_index_branch=False) -> ()");
// Apply repetition penalties to logits in-place.
ops.def(