[M3] Enable FP8 sparse GQA (#45744)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Thien Tran
2026-06-16 21:38:03 -07:00
committed by GitHub
co-authored by Yongye Zhu
parent d78650cf97
commit 4c62663315
10 changed files with 179 additions and 178 deletions
+8 -10
View File
@@ -32,19 +32,17 @@ message(STATUS "fmha_sm100 is available at ${fmha_sm100_SOURCE_DIR}")
add_custom_target(fmha_sm100)
set(FMHA_SM100_PY_ROOT "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100")
install(FILES
"${fmha_sm100_SOURCE_DIR}/python/fmha_sm100/__init__.py"
"${fmha_sm100_SOURCE_DIR}/python/fmha_sm100/sparse.py"
"${FMHA_SM100_PY_ROOT}/__init__.py"
"${FMHA_SM100_PY_ROOT}/sparse.py"
DESTINATION vllm/third_party/fmha_sm100
COMPONENT fmha_sm100)
install(DIRECTORY "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100/cute/"
install(DIRECTORY "${FMHA_SM100_PY_ROOT}/cute/"
DESTINATION vllm/third_party/fmha_sm100/cute
COMPONENT fmha_sm100
FILES_MATCHING
REGEX "/__pycache__(/.*)?$" EXCLUDE
REGEX ".*\\.pyc$" EXCLUDE
PATTERN "example.py" EXCLUDE
PATTERN "test_*.py" EXCLUDE
PATTERN "*.py"
PATTERN "build_k2q_csr.cu")
PATTERN "__pycache__" EXCLUDE
PATTERN "*.pyc" EXCLUDE
PATTERN ".git*" EXCLUDE)
@@ -58,8 +58,15 @@
#include "../cuda_compat.h"
#include "../type_convert.cuh"
#include "../attention/dtype_fp8.cuh"
#include "dispatch_utils.h"
#ifdef USE_ROCM
#include "../quantization/w8a8/fp8/amd/quant_utils.cuh"
#else
#include "../quantization/w8a8/fp8/nvidia/quant_utils.cuh"
#endif
#ifndef FINAL_MASK
#ifdef USE_ROCM
#define FINAL_MASK 0xffffffffffffffffULL
@@ -186,6 +193,21 @@ __device__ __forceinline__ void storeElems(
*reinterpret_cast<uint2*>(dst) = v;
}
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
__device__ __forceinline__ void storeCacheElems(
cache_t* __restrict__ dst, float const (&elems)[kElemsPerLane]) {
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
// kAuto means unquantized KV cache here: cache_t == scalar_t, so store the
// model dtype directly. FP8 cache dtypes use the conversion path below.
storeElems<scalar_t>(reinterpret_cast<scalar_t*>(dst), elems);
} else {
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
dst[i] = fp8::scaled_convert<cache_t, float, kv_dt>(elems[i], 1.0f);
}
}
}
// ────────────────────────────────────────────────────────────────────────────
// Kernel
// ────────────────────────────────────────────────────────────────────────────
@@ -202,7 +224,8 @@ __device__ __forceinline__ void storeElems(
// V : nkv only if kInsertKV (V-cache insert; no warps in dense)
// IQ: niq only if kIsSparse (norm+RoPE)
// IK: 1 only if kIsSparse (norm+RoPE; +index-cache insert)
template <typename scalar_t, bool kIsSparse, bool kInsertKV>
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt,
bool kIsSparse, bool kInsertKV>
__global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
scalar_t* __restrict__ qkv, // [N, qkv_row] in/out (packs index if sparse)
scalar_t* __restrict__ q_out, // [N, nq*128] contiguous, or nullptr
@@ -215,7 +238,7 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
int64_t const* __restrict__ positions, // [N] i64
int64_t const* __restrict__ slot_mapping, // main K/V slots or nullptr
int64_t const* __restrict__ index_slot_mapping, // index K slots/nullptr
scalar_t* __restrict__ kv_cache, // [nb,2,bs,nkv,128] or nullptr
cache_t* __restrict__ kv_cache, // [nb,2,bs,nkv,128] or nullptr
scalar_t* __restrict__ index_cache, // [nb*bs, 128] or nullptr
float const eps, int const rotary_dim, int const num_tokens, int const nq,
int const nkv, int const niq, int const block_size,
@@ -355,7 +378,8 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
int const kv = isK ? 0 : 1;
int64_t const off =
b * kv_s_block + kv * kv_s_kv + t * kv_s_token + head * kv_s_head;
storeElems<scalar_t>(kv_cache + off + dim_base, elems);
storeCacheElems<scalar_t, cache_t, kv_dt>(kv_cache + off + dim_base,
elems);
}
}
}
@@ -373,13 +397,13 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
// ────────────────────────────────────────────────────────────────────────────
// Launch wrapper
// ────────────────────────────────────────────────────────────────────────────
template <typename scalar_t>
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
void launchFusedMiniMaxM3(scalar_t* qkv, scalar_t* q_out, scalar_t* index_q_out,
scalar_t const* q_norm_w, scalar_t const* k_norm_w,
scalar_t const* iq_norm_w, scalar_t const* ik_norm_w,
scalar_t const* cos_sin_cache,
int64_t const* positions, int64_t const* slot_mapping,
int64_t const* index_slot_mapping, scalar_t* kv_cache,
int64_t const* index_slot_mapping, cache_t* kv_cache,
scalar_t* index_cache, float const eps,
int const rotary_dim, int const num_tokens,
int const nq, int const nkv, int const niq,
@@ -419,7 +443,8 @@ void launchFusedMiniMaxM3(scalar_t* qkv, scalar_t* q_out, scalar_t* index_q_out,
#define LAUNCH(IS_SPARSE, INSERT) \
cudaLaunchKernelEx( \
&config, \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, IS_SPARSE, INSERT>, \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, cache_t, kv_dt, \
IS_SPARSE, INSERT>, \
qkv, q_out, index_q_out, q_norm_w, k_norm_w, iq_norm_w, ik_norm_w, \
cos_sin_cache, positions, slot_mapping, index_slot_mapping, kv_cache, \
index_cache, eps, rotary_dim, num_tokens, nq, nkv, niq, block_size, \
@@ -428,7 +453,8 @@ void launchFusedMiniMaxM3(scalar_t* qkv, scalar_t* q_out, scalar_t* index_q_out,
// ROCm: standard kernel launch syntax (no PDL/stream serialization).
// clang-format off
#define LAUNCH(IS_SPARSE, INSERT) \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, IS_SPARSE, INSERT> \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, cache_t, kv_dt, \
IS_SPARSE, INSERT> \
<<<grid, kBlockSize, 0, stream>>>( \
qkv, q_out, index_q_out, q_norm_w, k_norm_w, iq_norm_w, \
ik_norm_w, cos_sin_cache, positions, slot_mapping, \
@@ -455,6 +481,33 @@ void launchFusedMiniMaxM3(scalar_t* qkv, scalar_t* q_out, scalar_t* index_q_out,
} // namespace minimax_m3_fused_ops
} // namespace vllm
#define CALL_FUSED_MINIMAX_M3(_RAW_T, CACHE_T, KV_DTYPE) \
vllm::minimax_m3_fused_ops::launchFusedMiniMaxM3<st, CACHE_T, KV_DTYPE>( \
reinterpret_cast<st*>(qkv.data_ptr()), \
q_out.has_value() ? reinterpret_cast<st*>(q_out->data_ptr()) : nullptr, \
index_q_out.has_value() ? reinterpret_cast<st*>(index_q_out->data_ptr()) \
: nullptr, \
reinterpret_cast<st const*>(q_norm_weight.data_ptr()), \
reinterpret_cast<st const*>(k_norm_weight.data_ptr()), \
has_index ? reinterpret_cast<st const*>(index_q_norm_weight->data_ptr()) \
: nullptr, \
has_index ? reinterpret_cast<st const*>(index_k_norm_weight->data_ptr()) \
: nullptr, \
reinterpret_cast<st const*>(cos_sin_cache.data_ptr()), \
reinterpret_cast<int64_t const*>(positions.data_ptr()), \
insert_kv ? reinterpret_cast<int64_t const*>(slot_mapping->data_ptr()) \
: nullptr, \
insert_kv ? reinterpret_cast<int64_t const*>( \
effective_index_slot_mapping->data_ptr()) \
: nullptr, \
insert_kv ? reinterpret_cast<CACHE_T*>(kv_cache->data_ptr()) : nullptr, \
(insert_kv && has_index) \
? reinterpret_cast<st*>(index_cache->data_ptr()) \
: nullptr, \
static_cast<float>(eps), static_cast<int>(rotary_dim), num_tokens, nq, \
nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_kv, kv_s_token, \
kv_s_head, has_index, insert_kv, stream)
// ────────────────────────────────────────────────────────────────────────────
// Torch op wrapper
// ────────────────────────────────────────────────────────────────────────────
@@ -475,9 +528,14 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
int64_t block_size,
std::optional<torch::stable::Tensor> q_out, // [N, nq*128] contiguous
std::optional<torch::stable::Tensor>
index_q_out) { // [N, niq*128] contiguous
index_q_out, // [N, niq*128] contiguous
const std::string& kv_cache_dtype) {
STD_TORCH_CHECK(qkv.is_cuda() && qkv.is_contiguous(),
"qkv must be contiguous CUDA");
STD_TORCH_CHECK(
qkv.scalar_type() == torch::headeronly::ScalarType::Half ||
qkv.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"qkv must be float16 or bfloat16");
STD_TORCH_CHECK(
positions.is_cuda() &&
positions.scalar_type() == torch::headeronly::ScalarType::Long,
@@ -510,6 +568,8 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
// (1 head)]) right after [q|k|v] in the same row; the dense layer does not.
bool const has_index = niq > 0;
bool const insert_kv = kv_cache.has_value();
vllm::Fp8KVCacheDataType const kv_dt =
vllm::get_fp8_kv_cache_data_type(kv_cache_dtype);
int const kHeadDim = vllm::minimax_m3_fused_ops::kHeadDim;
int const expected_row =
(nq + 2 * nkv + (has_index ? niq + 1 : 0)) * kHeadDim;
@@ -552,8 +612,14 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
torch::headeronly::ScalarType::Long &&
index_slot_mapping->numel() == slot_mapping->numel()),
"index_slot_mapping must be int64 CUDA with slot_mapping length");
STD_TORCH_CHECK(kv_cache->scalar_type() == qkv.scalar_type(),
"kv_cache dtype must match qkv (bf16 cache only)");
if (kv_dt == vllm::Fp8KVCacheDataType::kAuto) {
STD_TORCH_CHECK(kv_cache->scalar_type() == qkv.scalar_type(),
"auto kv_cache dtype must match qkv");
} else {
STD_TORCH_CHECK(
kv_cache->scalar_type() == torch::headeronly::ScalarType::Byte,
"fp8 kv_cache must use uint8 storage");
}
STD_TORCH_CHECK(index_cache.has_value() &&
index_cache->scalar_type() == qkv.scalar_type(),
"insert mode requires matching index_cache");
@@ -601,35 +667,9 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
VLLM_STABLE_DISPATCH_HALF_TYPES(
qkv.scalar_type(), "fused_minimax_m3_qknorm_rope_kv_insert", [&] {
using st = scalar_t;
vllm::minimax_m3_fused_ops::launchFusedMiniMaxM3<st>(
reinterpret_cast<st*>(qkv.data_ptr()),
q_out.has_value() ? reinterpret_cast<st*>(q_out->data_ptr())
: nullptr,
index_q_out.has_value()
? reinterpret_cast<st*>(index_q_out->data_ptr())
: nullptr,
reinterpret_cast<st const*>(q_norm_weight.data_ptr()),
reinterpret_cast<st const*>(k_norm_weight.data_ptr()),
has_index
? reinterpret_cast<st const*>(index_q_norm_weight->data_ptr())
: nullptr,
has_index
? reinterpret_cast<st const*>(index_k_norm_weight->data_ptr())
: nullptr,
reinterpret_cast<st const*>(cos_sin_cache.data_ptr()),
reinterpret_cast<int64_t const*>(positions.data_ptr()),
insert_kv
? reinterpret_cast<int64_t const*>(slot_mapping->data_ptr())
: nullptr,
insert_kv ? reinterpret_cast<int64_t const*>(
effective_index_slot_mapping->data_ptr())
: nullptr,
insert_kv ? reinterpret_cast<st*>(kv_cache->data_ptr()) : nullptr,
(insert_kv && has_index)
? reinterpret_cast<st*>(index_cache->data_ptr())
: nullptr,
static_cast<float>(eps), static_cast<int>(rotary_dim), num_tokens,
nq, nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_kv,
kv_s_token, kv_s_head, has_index, insert_kv, stream);
DISPATCH_BY_KV_CACHE_DTYPE(qkv.scalar_type(), kv_cache_dtype,
CALL_FUSED_MINIMAX_M3);
});
}
#undef CALL_FUSED_MINIMAX_M3
+5 -1
View File
@@ -3,6 +3,9 @@
#include <torch/csrc/stable/library.h>
#include <torch/csrc/stable/tensor.h>
#include <optional>
#include <string>
void per_token_group_quant_fp8(const torch::stable::Tensor& input,
torch::stable::Tensor& output_q,
torch::stable::Tensor& output_s,
@@ -297,7 +300,8 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
std::optional<torch::stable::Tensor> kv_cache,
std::optional<torch::stable::Tensor> index_cache, int64_t block_size,
std::optional<torch::stable::Tensor> q_out,
std::optional<torch::stable::Tensor> index_q_out);
std::optional<torch::stable::Tensor> index_q_out,
const std::string& kv_cache_dtype);
// Sampler kernels (shared CUDA/ROCm)
void apply_repetition_penalties_(
+2 -1
View File
@@ -471,7 +471,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"int num_index_heads, "
"Tensor? slot_mapping, Tensor? index_slot_mapping, "
"Tensor!? kv_cache, Tensor!? index_cache, "
"int block_size, Tensor!? q_out, Tensor!? index_q_out) -> ()");
"int block_size, Tensor!? q_out, Tensor!? index_q_out, "
"str kv_cache_dtype) -> ()");
// Apply repetition penalties to logits in-place.
ops.def(
+1 -1
View File
@@ -1168,7 +1168,7 @@ package_data = {
"third_party/deep_gemm/include/**/*.h",
"third_party/deep_gemm/include/**/*.hpp",
# fmha_sm100 sparse CuTe-DSL helper kernels (vendored via cmake)
"third_party/fmha_sm100/cute/src/sm100/build_k2q_csr/build_k2q_csr.cu",
"third_party/fmha_sm100/cute/**/*.cu",
]
}
@@ -106,7 +106,16 @@ def test_dense_norm_rope(num_tokens, num_heads, num_kv_heads):
qkv_orig = qkv.clone()
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv, q_w, k_w, cos_sin, positions, num_heads, num_kv_heads, ROTARY_DIM, eps
qkv,
q_w,
k_w,
cos_sin,
positions,
num_heads,
num_kv_heads,
ROTARY_DIM,
eps,
kv_cache_dtype="auto",
)
q_out, k_out, v_out = qkv.split([qsz, kvsz, kvsz], dim=-1)
@@ -133,7 +142,8 @@ def test_dense_norm_rope(num_tokens, num_heads, num_kv_heads):
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
@pytest.mark.parametrize("block_size", [16, 64])
def test_sparse_full(num_tokens, block_size):
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
torch.manual_seed(1)
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
base, max_pos = 5_000_000.0, 4096
@@ -158,8 +168,15 @@ def test_sparse_full(num_tokens, block_size):
splits = [qsz, kvsz, kvsz, iqsz, iksz]
num_blocks = (num_tokens + block_size - 1) // block_size + 1
kv_cache_storage_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
kv_cache = torch.zeros(
num_blocks, 2, block_size, num_kv_heads, HEAD_DIM, dtype=dtype, device=device
num_blocks,
2,
block_size,
num_kv_heads,
HEAD_DIM,
dtype=kv_cache_storage_dtype,
device=device,
)
index_cache = torch.zeros(
num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
@@ -195,11 +212,12 @@ def test_sparse_full(num_tokens, block_size):
block_size,
q_out,
index_q,
kv_cache_dtype,
)
# ── norm+rope parity. q/index_q land in their gather buffers; k/index_k are
# rewritten in place inside qkv. ──
_, k_out, _, _, index_k = qkv.split(splits, dim=-1)
_, k_out, v_out, _, index_k = qkv.split(splits, dim=-1)
q_in, k_in, v_in, iq_orig, ik_orig = qkv_orig.split(splits, dim=-1)
q_ref = norm_rope_ref(
q_in.view(num_tokens, num_heads, HEAD_DIM), q_w, positions, cos_sin, eps
@@ -230,15 +248,33 @@ def test_sparse_full(num_tokens, block_size):
# ── Cache inserts. ──
# Main cache layout is [num_blocks, 2, block_size, num_kv_heads, head_dim]
# (the K/V axis sits *before* block_size); index cache is [nb, bs, head_dim].
idx_flat = index_cache.view(num_blocks * block_size, HEAD_DIM)
k_ref_h = k_ref.view(num_tokens, num_kv_heads, HEAD_DIM)
v_ref_h = v_in.view(num_tokens, num_kv_heads, HEAD_DIM) # v is raw (no norm/rope)
for t in range(num_tokens):
s = slot_mapping[t].item()
b, pos = s // block_size, s % block_size
torch.testing.assert_close(
kv_cache[b, 0, pos], k_ref_h[t], rtol=1e-2, atol=1e-2
if kv_cache_dtype == "fp8":
expected_kv_cache = torch.zeros_like(kv_cache)
scale = torch.ones((), device=device)
ops.reshape_and_cache_flash(
k_out.view(num_tokens, num_kv_heads, HEAD_DIM),
v_out.view(num_tokens, num_kv_heads, HEAD_DIM),
expected_kv_cache[:, 0],
expected_kv_cache[:, 1],
slot_mapping,
kv_cache_dtype,
scale,
scale,
)
torch.testing.assert_close(kv_cache[b, 1, pos], v_ref_h[t], rtol=0, atol=0)
index_s = index_slot_mapping[t].item()
torch.testing.assert_close(idx_flat[index_s], ik_ref[t], rtol=1e-2, atol=1e-2)
torch.testing.assert_close(kv_cache, expected_kv_cache, rtol=0, atol=0)
else:
for t in range(num_tokens):
s = slot_mapping[t].item()
b, pos = s // block_size, s % block_size
torch.testing.assert_close(
kv_cache[b, 0, pos], k_ref_h[t], rtol=1e-2, atol=1e-2
)
torch.testing.assert_close(kv_cache[b, 1, pos], v_ref_h[t], rtol=0, atol=0)
expected_index_cache = torch.zeros_like(index_cache).view(-1, HEAD_DIM)
expected_index_cache[index_slot_mapping] = index_k
torch.testing.assert_close(
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
)
+5 -2
View File
@@ -2634,6 +2634,7 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
block_size: int = 0,
q_out: torch.Tensor | None = None,
index_q_out: torch.Tensor | None = None,
kv_cache_dtype: str = "auto",
) -> None:
"""Fused MiniMax-M3 attention pre-processing (in-place).
@@ -2645,8 +2646,9 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
index_k]`` — the index branch is read straight out of ``qkv``.
When ``kv_cache`` is given (sparse serving), also scatter-inserts the
normed/roped k & v into the paged bf16 KV cache by ``slot_mapping`` and the
index key into ``index_cache`` by ``index_slot_mapping``. If
normed/roped k & v into the paged KV cache by ``slot_mapping`` and the
index key into ``index_cache`` by ``index_slot_mapping``. ``kv_cache_dtype``
selects the cache storage/conversion path. If
``index_slot_mapping`` is omitted, ``slot_mapping`` is used for both caches.
If ``q_out`` / ``index_q_out`` (contiguous ``[N, nq*128]`` / ``[N,
@@ -2675,6 +2677,7 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
block_size,
q_out,
index_q_out,
kv_cache_dtype,
)
+7 -63
View File
@@ -426,6 +426,7 @@ class MiniMaxM3Attention(nn.Module):
self.num_kv_heads,
self.rotary_emb.rotary_dim,
self.q_norm.variance_epsilon,
kv_cache_dtype="auto",
)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
attn_output = self.attn(q, k, v)
@@ -533,11 +534,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
self.kv_cache_dtype, vllm_config.model_config
)
# fp8 main-K/V cache: the fused qknorm+rope+kv-insert op is bf16-cache-only
# (asserts kv_cache dtype == qkv), so on the fp8 path we run it in
# norm+rope-only mode and write the cache via the fp8-capable
# reshape_and_cache_flash in _insert_kv. (index cache stays bf16.)
self._fp8_kv = "fp8" in self.kv_cache_dtype
self.attn_backend = MiniMaxM3SparseBackend
# Indexer and main attention are separate impls. On ROCm the SM100 gate
@@ -592,37 +588,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
def _insert_kv(
self,
key: torch.Tensor,
value: torch.Tensor,
index_key: torch.Tensor,
main_slot_mapping: torch.Tensor,
index_slot_mapping: torch.Tensor,
) -> None:
"""Write main K/V (fp8-quantizing) and index-K into their paged caches.
Used only on the fp8-KV path: the fused #20 op is bf16-cache-only, so it
runs in norm+rope-only mode and the (already normed/roped) k/v/index_k are
written here via ``reshape_and_cache_flash`` (which honors kv_cache_dtype,
unit scale -- matching the fp8 read path added in #33). Mirrors the
pre-#20 unfused insert. The index cache stays bf16 (no quant).
"""
key_cache, value_cache = self.kv_cache.unbind(1)
scale = torch.ones((), device=key.device)
ops.reshape_and_cache_flash(
key.view(-1, self.num_kv_heads, self.head_dim),
value.view(-1, self.num_kv_heads, self.head_dim),
key_cache,
value_cache,
main_slot_mapping,
self.kv_cache_dtype,
scale,
scale,
)
idx_cache = self.indexer.index_cache.kv_cache.view(-1, self.idx_head_dim)
idx_cache[index_slot_mapping] = index_key.to(idx_cache.dtype)
def forward(
self,
positions: torch.Tensor,
@@ -636,11 +601,9 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
# of the single fused ``qkv`` tensor. Once the paged caches are bound the
# kernel also inserts k/v and the index key into them (each with its own
# slot_mapping); the memory-profiling run (caches unbound, no slot_mapping)
# short-circuits to zeros below. Replaces the
# q_norm/k_norm/rotary_emb/index_*_norm/index_rotary_emb/_insert_kv chain.
# (#20 fused_minimax_m3_qknorm_rope_kv_insert; HIP/CDNA path. The main and
# index slot mappings are read from the forward context's slot_mapping
# dict, matching the breakable-cudagraph path -- see nvidia/model.py.)
# short-circuits to zeros below. The main and index slot mappings are read
# from the forward context's slot_mapping dict, matching the
# breakable-cudagraph path -- see nvidia/model.py.
cos_sin_cache = self.rotary_emb.cos_sin_cache
rotary_dim = self.rotary_emb.rotary_dim
eps = self.q_norm.variance_epsilon
@@ -658,12 +621,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
q = qkv.new_empty((num_tokens, self.q_size))
index_q = qkv.new_empty((num_tokens, self.index_q_size))
# On the fp8-KV path the fused op cannot write the (fp8) cache, so pass
# kv_cache/index_cache = None -> insert_kv=False (norm+rope only): it still
# de-interleaves q/index_q and rewrites the normed/roped k & index_k in
# place in qkv, leaving v raw (correct -- v is never normed/roped). We then
# write the cache via _insert_kv below.
insert_via_fused = not self._fp8_kv
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
@@ -679,26 +636,13 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.num_idx_heads,
main_slot_mapping,
index_slot_mapping,
self.kv_cache if insert_via_fused else None,
self.indexer.index_cache.kv_cache if insert_via_fused else None,
self.kv_cache,
self.indexer.index_cache.kv_cache,
self.kv_cache.size(2), # paged-cache block size
q,
index_q,
self.kv_cache_dtype,
)
if not insert_via_fused:
# Extract the normed/roped k, raw v, normed/roped index_k from qkv
# ([q | k | v | index_q | index_k], all head_dim=128) and fp8-insert.
kv = self.num_kv_heads * self.head_dim
# These are strided views into qkv (row stride = full qkv width), but
# their last dim is contiguous, so `_insert_kv`'s `.view(-1, nkv,
# head_dim)` works on them and `reshape_and_cache_flash` honors the
# input stride -- no `.contiguous()` needed (verified bit-identical;
# avoids a [N, kv] copy per step on the fp8-KV path).
k = qkv[:, self.q_size : self.q_size + kv]
v = qkv[:, self.q_size + kv : self.q_size + 2 * kv]
ik0 = self.q_size + 2 * kv + self.index_q_size
index_k = qkv[:, ik0 : ik0 + self.num_idx_heads * self.idx_head_dim]
self._insert_kv(k, v, index_k, main_slot_mapping, index_slot_mapping)
output = torch.empty_like(q)
attn_output = self._run_attention(q, index_q, output)
@@ -22,6 +22,7 @@ import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.models.minimax_m3.common.ops.sparse_attn import (
SPARSE_BLOCK_SIZE,
minimax_m3_sparse_attn,
@@ -44,6 +45,8 @@ from vllm.v1.attention.backends.utils import (
)
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
logger = init_logger(__name__)
class MiniMaxM3SparseBackend(AttentionBackend):
"""Block-sparse GQA backend for MiniMax M3 sparse attention layers."""
@@ -385,16 +388,25 @@ def select_main_impl_cls(
) -> type[MiniMaxM3SparseImpl]:
"""Pick the main attend impl off the main KV-cache dtype.
bf16 on Blackwell (SM100) uses the MSA attend; fp8 or non-Blackwell falls
Blackwell (SM100) uses the MSA attend for supported top-k block counts
when the KV cache is BF16 or FP8 E4M3; non-Blackwell and FP8 E5M2 fall
back to Triton. The MSA module is imported lazily so AMD/non-SM100 never
import fmha_sm100.
"""
if (
use_msa = (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and topk_blocks in (4, 8, 16, 32)
and not is_quantized_kv_cache(kv_cache_dtype)
):
and kv_cache_dtype != "fp8_e5m2"
)
selected = "MSA" if use_msa else "Triton"
logger.info_once(
"MiniMax M3 sparse attention selected %s (kv_cache_dtype=%s, topk_blocks=%s)",
selected,
kv_cache_dtype,
topk_blocks,
)
if use_msa:
from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
MiniMaxM3SparseMSAImpl,
)
+6 -43
View File
@@ -66,10 +66,7 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model
from vllm.models.minimax_m3.common.indexer import (
MiniMaxM3Indexer,
MiniMaxM3IndexerMetadata,
)
from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer
from vllm.models.minimax_m3.common.mm_preprocess import (
MiniMaxM3VLDummyInputsBuilder,
MiniMaxM3VLMultiModalProcessor,
@@ -78,7 +75,6 @@ from vllm.models.minimax_m3.common.mm_preprocess import (
from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseBackend,
MiniMaxM3SparseImpl,
MiniMaxM3SparseMetadata,
select_main_impl_cls,
)
from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel
@@ -375,6 +371,7 @@ class MiniMaxM3Attention(nn.Module):
self.num_kv_heads,
self.rotary_emb.rotary_dim,
self.q_norm.variance_epsilon,
kv_cache_dtype="auto",
)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
attn_output = self.attn(q, k, v)
@@ -545,40 +542,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
def _insert_kv(
self, key: torch.Tensor, value: torch.Tensor, index_key: torch.Tensor
) -> None:
"""Write main K/V and index-K into their paged caches.
No-op during the profiling run, where caches are not yet bound and
``attn_metadata`` is None.
"""
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
return
main_meta = attn_metadata[self.layer_name]
index_meta = attn_metadata[self.indexer.index_cache.prefix]
assert isinstance(main_meta, MiniMaxM3SparseMetadata)
assert isinstance(index_meta, MiniMaxM3IndexerMetadata)
# Identity scale: unused for the bf16 cache, required arg of the op.
key_cache, value_cache = self.kv_cache.unbind(1)
scale = torch.ones((), device=key.device)
ops.reshape_and_cache_flash(
key.view(-1, self.num_kv_heads, self.head_dim),
value.view(-1, self.num_kv_heads, self.head_dim),
key_cache,
value_cache,
main_meta.slot_mapping,
self.kv_cache_dtype,
scale,
scale,
)
# Index-key cache: single vector per token, scatter by slot.
idx_cache = self.indexer.index_cache.kv_cache.view(-1, self.idx_head_dim)
idx_cache[index_meta.slot_mapping] = index_key.to(idx_cache.dtype)
def forward(
self,
positions: torch.Tensor,
@@ -592,10 +555,9 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
# of the single fused ``qkv`` tensor (the "5 results"). Once the paged
# caches are bound the kernel also inserts k/v and the index key into
# them; the initial memory-profiling run (caches unbound, no slot_mapping)
# short-circuits to zeros below. Replaces the
# q_norm/k_norm/rotary_emb/index_*_norm/index_rotary_emb/_insert_kv
# sequence. k/v and index_k are rewritten in place inside qkv (and
# scatter-inserted into the caches); q and index_q are de-interleaved
# short-circuits to zeros below. k/v and index_k are rewritten in place
# inside qkv (and scatter-inserted into the caches); q and index_q are
# de-interleaved
# straight into the dedicated contiguous ``q``/``index_q`` buffers below.
cos_sin_cache = self.rotary_emb.cos_sin_cache
@@ -635,6 +597,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.kv_cache.size(2), # paged-cache block size
q,
index_q,
self.kv_cache_dtype,
)
output = torch.empty_like(q)