[CPU] Add AMX-only high-performance MLA backend for DeepSeek V2/V3/R1 (#52616)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-08-19 10:28:52 +08:00
committed by GitHub
parent f485081e8b
commit e4d61d0d22
20 changed files with 4156 additions and 6 deletions
+4
View File
@@ -12,6 +12,9 @@ steps:
- vllm/_custom_ops.py
- tests/kernels/attention/test_cpu_attn.py
- tests/v1/attention/test_group_head_counts.py
- tests/kernels/attention/test_amx_mla.py
- vllm/v1/attention/backends/mla/amx_mla.py
- vllm/model_executor/layers/attention/mla_attention.py
- tests/kernels/moe/test_cpu_fused_moe.py
- tests/kernels/moe/test_cpu_quant_fused_moe.py
- tests/kernels/test_onednn.py
@@ -29,6 +32,7 @@ steps:
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
pytest -x -v -s tests/v1/attention/test_group_head_counts.py
pytest -x -v -s tests/kernels/attention/test_amx_mla.py
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
+5 -1
View File
@@ -519,7 +519,11 @@ if (ENABLE_X86_ISA)
"csrc/cpu/sgl-kernels/moe.cpp"
"csrc/cpu/sgl-kernels/moe_int8.cpp"
"csrc/cpu/sgl-kernels/moe_int4.cpp"
"csrc/cpu/sgl-kernels/moe_fp8.cpp")
"csrc/cpu/sgl-kernels/moe_fp8.cpp"
"csrc/cpu/sgl-kernels/bmm.cpp"
"csrc/cpu/sgl-kernels/decode.cpp"
"csrc/cpu/sgl-kernels/extend.cpp"
"csrc/cpu/sgl-kernels/mla_cache.cpp")
set(VLLM_EXT_SRC_AVX512
"csrc/cpu/sgl-kernels/fla.cpp"
+208
View File
@@ -0,0 +1,208 @@
// Adapted from
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
// clang-format off
#include "common.h"
#include "gemm.h"
#include "vec.h"
namespace {
template <typename scalar_t, typename packed_t>
void bmm_kernel_impl(
scalar_t* __restrict__ out,
const scalar_t* __restrict__ mat1,
const packed_t* __restrict__ mat2,
int64_t B,
int64_t M,
int64_t N,
int64_t K,
int64_t mat1_strideB,
int64_t mat1_strideM,
int64_t out_strideB,
int64_t out_strideM,
float scale = 0.f) {
constexpr int64_t BLOCK_M = block_size_m();
constexpr int64_t BLOCK_N = block_size_n();
const int64_t MB = div_up(M, BLOCK_M);
const int64_t NB = div_up(N, BLOCK_N);
// mat2 contiguous in [B, N, K]
int64_t mat2_strideB = N * K;
int64_t mat2_strideN = K;
const bool use_brgemm = can_use_brgemm<scalar_t>(M);
// parallel on [B, MB, NB]
at::parallel_for(0, B * MB * NB, 0, [&](int64_t begin, int64_t end) {
int64_t bs{0}, mb{0}, nb{0};
data_index_init(begin, bs, B, mb, MB, nb, NB);
// for brgemm, use float32 for accumulate
alignas(64) float Ctmp[BLOCK_M * BLOCK_N];
for (int i = begin; i < end; ++i) {
UNUSED(i);
int mb_start = mb * BLOCK_M;
int mb_size = std::min(M - mb_start, BLOCK_M);
int nb_start = nb * BLOCK_N;
int nb_size = std::min(N - nb_start, BLOCK_N);
tinygemm_kernel<scalar_t>(
/* A */ mat1 + bs * mat1_strideB + mb_start * mat1_strideM,
/* B */ mat2 + bs * mat2_strideB + nb_start * mat2_strideN /* nb * BLOCK_N * K */,
/* C */ out + bs * out_strideB + mb_start * out_strideM + nb_start,
/* Ctmp*/ Ctmp,
/* M */ mb_size,
/* N */ nb_size,
/* K */ K,
/* lda */ mat1_strideM,
/* ldb */ nb_size,
/* ldc */ out_strideM,
/* brg */ use_brgemm);
// move to the next index
data_index_step(bs, B, mb, MB, nb, NB);
}
if (use_brgemm) {
at::native::cpublas::brgemm_release();
}
});
}
template <>
void bmm_kernel_impl(
at::BFloat16* __restrict__ out,
const at::BFloat16* __restrict__ mat1,
const at::Float8_e4m3fn* __restrict__ mat2,
int64_t B,
int64_t M,
int64_t N,
int64_t K,
int64_t mat1_strideB,
int64_t mat1_strideM,
int64_t out_strideB,
int64_t out_strideM,
float scale) {
constexpr int64_t BLOCK_M = block_size_m();
constexpr int64_t BLOCK_N = block_size_n();
const int64_t MB = div_up(M, BLOCK_M);
const int64_t NB = div_up(N, BLOCK_N);
// mat2 contiguous in [B, N, K]
int64_t mat2_strideB = N * K;
int64_t mat2_strideN = K;
const bool use_brgemm = can_use_brgemm<at::BFloat16>(M);
// parallel on [B, MB, NB]
parallel_2d(B * MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
// for brgemm, use float32 for accumulate
alignas(64) float Ctmp[BLOCK_M * BLOCK_N];
// for brgemm when mat2 is float8_e4m3
alignas(64) at::BFloat16 Btmp[BLOCK_N * BLOCK_K];
loop_2d<at::Float8_e4m3fn>(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
int64_t bs = mb / MB;
int64_t mb_start = (mb % MB) * BLOCK_M;
int64_t mb_size = std::min(M - mb_start, BLOCK_M);
int64_t nb_start = nb * BLOCK_N;
int64_t nb_size = std::min(N - nb_start, BLOCK_N);
tinygemm_kernel(
/* A */ mat1 + bs * mat1_strideB + mb_start * mat1_strideM,
/* B */ mat2 + bs * mat2_strideB + nb_start * mat2_strideN /* nb * BLOCK_N * K */,
/* C */ out + bs * out_strideB + mb_start * out_strideM + nb_start,
/* Btmp*/ Btmp,
/* Ctmp*/ Ctmp,
/*scale*/ scale,
/* M */ mb_size,
/* N */ nb_size,
/* K */ K,
/* lda */ mat1_strideM,
/* ldb */ nb_size,
/* ldc */ out_strideM,
/* brg */ use_brgemm);
});
if (use_brgemm) {
at::native::cpublas::brgemm_release();
}
});
}
} // anonymous namespace
// mat1 : [B, M, K]
// mat2 : [B, N, K] or [B, OC, IC]
// out : [B, M, N]
// scale: [] 0-dim tensor for per tensor quant
//
void bmm_cpu(
at::Tensor& out, at::Tensor& mat1, at::Tensor& mat2, bool is_vnni, const std::optional<at::Tensor>& scale) {
auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2);
// input and out could be non-contiguous
// weight needs to be contiguous in [OC, IC] order
CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1);
CHECK_LAST_DIM_CONTIGUOUS_INPUT(out);
CHECK_INPUT(mat2);
CHECK_DIM(3, out);
CHECK_DIM(3, mat1);
CHECK_DIM(3, mat2);
int64_t B = mat1.size(0);
int64_t M = mat1.size(1);
int64_t N = mat2.size(1);
int64_t K = mat1.size(2);
const bool use_fp8_w8a16 = scale.has_value();
TORCH_CHECK(N % 32 == 0, "tinygemm requires N to be 32x.");
int64_t mat1_strideB = mat1.stride(0);
int64_t mat1_strideM = mat1.stride(1);
int64_t out_strideB = out.stride(0);
int64_t out_strideM = out.stride(1);
// check shapes
TORCH_CHECK(mat2.size(0) == B && mat2.size(2) == K, "bmm: mat2 shape mismatch!");
TORCH_CHECK(out.size(0) == B && out.size(1) == M, "bmm: out shape mismatch!");
if (!use_fp8_w8a16) {
AT_DISPATCH_REDUCED_FLOATING_TYPES(mat1.scalar_type(), "bmm_kernel_impl", [&] {
bmm_kernel_impl<scalar_t, scalar_t>(
out.data_ptr<scalar_t>(),
mat1.data_ptr<scalar_t>(),
packed_w.data_ptr<scalar_t>(),
B,
M,
N,
K,
mat1_strideB,
mat1_strideM,
out_strideB,
out_strideM);
});
} else { // fp8 bmm
float scale_val = 0.f;
auto scale_tensor = scale.value();
TORCH_CHECK(scale_tensor.ndimension() == 0, "bmm: expect scale to be 0-dim tensor.");
scale_val = scale_tensor.item<float>();
bmm_kernel_impl<at::BFloat16, at::Float8_e4m3fn>(
out.data_ptr<at::BFloat16>(),
mat1.data_ptr<at::BFloat16>(),
packed_w.data_ptr<at::Float8_e4m3fn>(),
B,
M,
N,
K,
mat1_strideB,
mat1_strideM,
out_strideB,
out_strideM,
scale_val);
}
}
File diff suppressed because it is too large Load Diff
+568
View File
@@ -0,0 +1,568 @@
// Adapted from
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
// clang-format off
#include "common.h"
#include "flash_attn.h"
#include "gemm.h"
namespace {
// [NOTE]: extend attention for CPU
// 1. BLOCK_M and BLOCK_N tuned for various seq lengths
// 2. can handle non-contiguous k_extend and v_extend
// 3. computes attention for prefix and extend separately
// 4. TODO: apply head dimension blocking to optimize GQA
// 5. optional tree mask for speculative decoding TARGET_VERIFY (EAGLE topk > 1):
// `tree_mask` is a flat [batches * qlen * qlen] bool tensor in
// TreeMaskMode::QLEN_ONLY layout, where qlen == extend_seq_lens[bs] ==
// max_len_extend (uniform across the batch, equal to draft_token_num).
// Row i = query draft token, column j = key draft token; true means query i
// may attend key j (each row marks self + ancestors + root). The committed
// prefix (stage 1) is implicitly fully visible to every draft token, which
// is why the mask only covers the qlen x qlen new-token block; the GPU
// FULL_MASK layout carries the prefix columns explicitly but they are
// all-true for EAGLE. When tree_mask is absent, stage 2 falls back to the
// plain causal mask (correct for non-spec extend and topk == 1 chains).
//
template <typename scalar_t, typename index_t, int BLOCK_M, int BLOCK_N>
void extend_attention_kernel_impl(
scalar_t* __restrict__ o_extend,
const scalar_t* __restrict__ q_extend,
const scalar_t* __restrict__ k_extend,
const scalar_t* __restrict__ v_extend,
const scalar_t* __restrict__ k_buffer,
const scalar_t* __restrict__ v_buffer,
const index_t* __restrict__ req_to_token,
const int64_t* __restrict__ req_pool_indices,
const int64_t* __restrict__ seq_lens,
const int64_t* __restrict__ encoder_lens,
const index_t* __restrict__ extend_seq_lens,
const index_t* __restrict__ extend_start_loc,
const void* __restrict__ buffer,
const scalar_t* __restrict__ sinks,
const bool* __restrict__ tree_mask,
int batches,
int num_heads,
int num_heads_kv,
int head_size,
int head_size_v,
int q_strideM,
int q_strideH,
int ke_strideN,
int ke_strideH,
int ve_strideN,
int ve_strideH,
int k_strideN,
int k_strideH,
int v_strideN,
int v_strideH,
float sm_scale,
int max_num_reqs,
int max_context_len,
int max_total_num_tokens,
int max_len_extend,
int buffer_size_per_thread,
int64_t sliding_window_size,
bool is_prefix_skipped,
bool is_cross_attn,
bool has_encoder_lens,
bool has_sink) {
// strides
const int o_strideM = num_heads * head_size_v;
const int o_strideH = head_size_v;
// we use same buffer for packed key and value
const int ldb_tmp = std::max(head_size, head_size_v);
const int num_groups = num_heads / num_heads_kv;
TORCH_CHECK(num_groups * num_heads_kv == num_heads);
// number of blocks along M
int MB = div_up(max_len_extend, BLOCK_M);
// parallel on [batches, num_heads, BM]
at::parallel_for(0, batches * num_heads * MB, 0, [&](int begin, int end) {
int bs{0}, head_id{0}, mb{0};
data_index_init(begin, bs, batches, head_id, num_heads, mb, MB);
int tid = at::get_thread_num();
// s_i: [BLOCK_M, BLOCK_N]
float* __restrict__ s_i = reinterpret_cast<float*>((char*)(buffer) + tid * buffer_size_per_thread);
// v_prime: [BLOCK_M, head_size_v]
float* __restrict__ v_prime = s_i + BLOCK_M * BLOCK_N;
// s_delta: [BLOCK_M, BLOCK_N]
scalar_t* __restrict__ s_delta = reinterpret_cast<scalar_t*>(v_prime + BLOCK_M * head_size_v);
// Btmp: [BLOCK_N, max(head_size, head_size_v)]
scalar_t* __restrict__ Btmp = reinterpret_cast<scalar_t*>(s_delta + BLOCK_M * BLOCK_N);
// init Btmp just once for each thread to prevent NaN
fill_stub(Btmp, 0.f, BLOCK_N * ldb_tmp);
fill_stub(s_delta, 0.f, BLOCK_M * BLOCK_N);
alignas(64) float s_prime[BLOCK_M];
alignas(64) float m_prime[BLOCK_M];
for (int i = begin; i < end; ++i) {
// seq_len = prefix + extend
int head_kv_id = head_id / num_groups;
int seq_len = seq_lens[bs];
int seq_len_extend = extend_seq_lens[bs];
int seq_len_prefix = seq_len - seq_len_extend;
int seq_extend_start_loc = extend_start_loc[bs];
int req_pool_id = req_pool_indices[bs];
int kv_offset = (has_encoder_lens && (!is_cross_attn)) ? encoder_lens[bs] : 0;
TORCH_CHECK(seq_len_prefix >= 0, "prefix len < 0!");
TORCH_CHECK(seq_len <= max_context_len, "seq_len out of scope!");
TORCH_CHECK(req_pool_id < max_num_reqs, "req_pool_id out of scope!");
if (is_prefix_skipped) {
TORCH_CHECK(seq_len_prefix == 0, "extend attention: expect seq_len_prefix to be 0, got ", seq_len_prefix);
}
if (tree_mask != nullptr) {
// QLEN_ONLY layout assumes a uniform qlen across the batch (TARGET_VERIFY)
TORCH_CHECK(
seq_len_extend == max_len_extend,
"extend attention: tree_mask requires uniform extend_seq_lens, got ",
seq_len_extend,
" vs ",
max_len_extend);
}
// offset and size in MB
int m = mb * BLOCK_M;
int m_size = std::min(BLOCK_M, seq_len_extend - m);
if (m_size <= 0) {
data_index_step(bs, batches, head_id, num_heads, mb, MB);
continue;
}
// get query
const scalar_t* __restrict__ q_ptr = q_extend + (seq_extend_start_loc + m) * q_strideM + head_id * q_strideH;
// init v', s' and m'
fill_stub(v_prime, 0.f, m_size * head_size_v);
fill_stub(s_prime, 0.f, m_size);
fill_stub(m_prime, -std::numeric_limits<scalar_t>::infinity(), m_size);
// stage 1: compute scores with prefix
int kv_start = 0;
int kv_end = is_cross_attn ? encoder_lens[bs] : seq_len_prefix;
for (int n = kv_start; n < kv_end; n += BLOCK_N) {
int n_size = std::min(BLOCK_N, kv_end - n);
// `n_size` is K in 2nd gemm, pad to TILE_K;
const int padded_n_size = div_up(n_size, TILE_K) * TILE_K;
// get key and pack
pack_vnni<scalar_t, index_t>(
/* dst */ Btmp,
/* src */ k_buffer + head_kv_id * k_strideH,
/* ind */ req_to_token + req_pool_id * max_context_len + n + kv_offset,
/* N */ n_size,
/* K */ head_size,
/* ld_src */ k_strideN,
/* ld_dst */ BLOCK_N);
// calculate s_i <- Q @ K
at::native::cpublas::brgemm(
/* M */ m_size,
/* N */ n_size,
/* K */ head_size,
/* lda */ q_strideM,
/* ldb */ BLOCK_N,
/* ldc */ BLOCK_N,
/* add_C */ false,
/* A */ q_ptr,
/* B */ Btmp,
/* C */ s_i);
for (int row = 0; row < m_size; ++row) {
if (sliding_window_size > 0) {
int last_col = seq_len_prefix + row + m - sliding_window_size + 1;
if (last_col >= n + n_size) {
continue;
}
fill_stub(s_i + row * BLOCK_N, -std::numeric_limits<float>::infinity(), last_col - n);
}
flash_attn_softmax<scalar_t, BLOCK_M, BLOCK_N>::apply(
s_i, s_delta, v_prime, s_prime, m_prime, m_size, n_size, padded_n_size, head_size_v, sm_scale, row);
}
// get value and pack
pack_vnni2<scalar_t>(
/* dst */ Btmp,
/* src */ v_buffer + head_kv_id * v_strideH,
/* ind */ req_to_token + req_pool_id * max_context_len + n + kv_offset,
/* K */ n_size,
/* N */ head_size_v,
/* ld_src */ v_strideN,
/* ld_dst */ head_size_v);
// calculate V' <- s_delta @ V + V'
at::native::cpublas::brgemm(
/* M */ m_size,
/* N */ head_size_v,
/* K */ padded_n_size, // n_size
/* lda */ BLOCK_N,
/* ldb */ head_size_v,
/* ldc */ head_size_v,
/* add_C */ true,
/* A */ s_delta,
/* B */ Btmp,
/* C */ v_prime);
} // loop with seq_len_prefix
if (!is_cross_attn) {
// stage 2: compute the triangle part
int num_keys = std::min(seq_len_extend, m + BLOCK_M);
for (int n = 0; n < num_keys; n += BLOCK_N) {
int n_size = std::min(BLOCK_N, num_keys - n);
// `n_size` is K in 2nd gemm, pad to TILE_K;
const int padded_n_size = div_up(n_size, TILE_K) * TILE_K;
// get key and pack
pack_vnni<scalar_t>(
/* dst */ Btmp,
/* src */ k_extend + (seq_extend_start_loc + n) * ke_strideN + head_kv_id * ke_strideH,
/* N */ n_size,
/* K */ head_size,
/* ld_src */ ke_strideN,
/* ld_dst */ BLOCK_N);
// calculate s_i <- Q @ K
at::native::cpublas::brgemm(
/* M */ m_size,
/* N */ n_size,
/* K */ head_size,
/* lda */ q_strideM,
/* ldb */ BLOCK_N,
/* ldc */ BLOCK_N,
/* add_C */ false,
/* A */ q_ptr,
/* B */ Btmp,
/* C */ s_i);
// apply tree mask (speculative TARGET_VERIFY) or causal mask
if (tree_mask != nullptr) {
// [Note] tree mask for EAGLE topk > 1 (TreeMaskMode::QLEN_ONLY).
// mask[bs][m + row][n + col] == false -> query draft token (m + row)
// may not attend key draft token (n + col); set the score to -inf
// before softmax. The tree mask subsumes the causal constraint:
// ancestors always precede descendants in the draft token ordering,
// so permitted keys satisfy j <= i and the causal `num_keys` bound
// above remains valid.
const bool* __restrict__ mask_base =
tree_mask + (static_cast<int64_t>(bs) * seq_len_extend + m) * seq_len_extend + n;
for (int row = 0; row < m_size; ++row) {
float* __restrict__ row_ptr = s_i + row * BLOCK_N;
const bool* __restrict__ mask_ptr = mask_base + static_cast<int64_t>(row) * seq_len_extend;
for (int col = 0; col < n_size; ++col) {
if (!mask_ptr[col]) {
row_ptr[col] = -std::numeric_limits<float>::infinity();
}
}
}
} else if (n + n_size - 1 > m) {
// apply causal mask
// [Note] condition to apply causal mask.
// Mask any block whose last key (n + n_size - 1) is strictly after the first query position (m), i.e. n +
// n_size - 1 > m. The original condition was `num_keys - n <= BLOCK_N` (last n-block only). That was
// correct when BLOCK_M <= BLOCK_N/2 because earlier n-blocks were guaranteed to contain only past keys.
// With BLOCK_M=512, BLOCK_N=768:
// BLOCK_M > BLOCK_N/2, so the first n-block can contain future keys.
// Example: m=512 (mb=1), num_keys=1024, first n-block covers keys [0, 768).
// Query row=0 is at position 512, so keys 513..767 are future and must be
// masked — but `num_keys - 0 = 1024 > BLOCK_N` skips masking entirely,
// producing wrong (non-causal) attention for rows 0..254 of this m-block.
for (int row = 0; row < m_size; ++row) {
int last_col = m + row - n;
// [Note] mask the entire row if last_col < 0.
// Clamp to -1: when n > m + row every key in this block is a future
// key, so the entire row should be masked. Without this clamp,
// last_col+1 <= 0 and fill_stub would write before row_ptr.
last_col = std::max(last_col, -1);
// fill [last_col + 1, n_size) to -inf
float* row_ptr = s_i + row * BLOCK_N;
fill_stub(row_ptr + last_col + 1, -std::numeric_limits<float>::infinity(), n_size - last_col - 1);
}
}
for (int row = 0; row < m_size; ++row) {
if (sliding_window_size > 0 && row + m + 1 >= n + sliding_window_size - 1 &&
row + m + 1 < n + sliding_window_size + n_size) {
fill_stub(
s_i + row * BLOCK_N, -std::numeric_limits<float>::infinity(), row + m - n - sliding_window_size + 1);
} else if (sliding_window_size > 0 && row + m + 1 >= n + sliding_window_size) {
continue;
}
flash_attn_softmax<scalar_t, BLOCK_M, BLOCK_N>::apply(
s_i, s_delta, v_prime, s_prime, m_prime, m_size, n_size, padded_n_size, head_size_v, sm_scale, row);
}
// get value and pack
pack_vnni2<scalar_t>(
/* dst */ Btmp,
/* src */ v_extend + (seq_extend_start_loc + n) * ve_strideN + head_kv_id * ve_strideH,
/* K */ n_size,
/* N */ head_size_v,
/* ld_src */ ve_strideN,
/* ld_dst */ head_size_v);
// calculate V' <- s_delta @ V + V'
at::native::cpublas::brgemm(
/* M */ m_size,
/* N */ head_size_v,
/* K */ padded_n_size, // n_size
/* lda */ BLOCK_N,
/* ldb */ head_size_v,
/* ldc */ head_size_v,
/* add_C */ true,
/* A */ s_delta,
/* B */ Btmp,
/* C */ v_prime);
} // loop with seq_len_extend
}
scalar_t* __restrict__ out_ptr = o_extend + (seq_extend_start_loc + m) * o_strideM + head_id * o_strideH;
for (int row = 0; row < m_size; ++row) {
if (has_sink) {
s_prime[row] += std::exp(sinks[head_id] - m_prime[row]);
}
float s = 1 / s_prime[row];
copy_stub<scalar_t>(out_ptr + row * o_strideM, v_prime + row * head_size_v, s, head_size_v);
}
// move to the next index
data_index_step(bs, batches, head_id, num_heads, mb, MB);
}
at::native::cpublas::brgemm_release();
});
}
} // anonymous namespace
template <int BLOCK_M, int BLOCK_N>
inline int resize_buffer(at::Tensor& buffer, int num_threads, int head_size, int head_size_v) {
static_assert(BLOCK_M <= BLOCK_N, "Make sure BLOCK_M <= BLOCK_N to prevent buffer overflows during causal masking");
const int size_per_thread =
/* s_i */ BLOCK_M * BLOCK_N * sizeof(float) +
/* v_prime */ BLOCK_M * head_size_v * sizeof(float) +
/* s_delta */ BLOCK_M * BLOCK_N * sizeof(uint16_t) +
/* Btmp */ BLOCK_N * std::max(head_size, head_size_v) * sizeof(uint16_t);
buffer.resize_({num_threads, size_per_thread});
return size_per_thread;
}
#define LAUNCH_EXTEND_ATTENTION_KERNEL(BLOCK_M, BLOCK_N) \
do { \
int sz = resize_buffer<BLOCK_M, BLOCK_N>(buffer, num_threads, head_size, head_size_v); \
\
extend_attention_kernel_impl<scalar_t, index_t, BLOCK_M, BLOCK_N>( \
o_extend.data_ptr<scalar_t>(), \
q_extend.data_ptr<scalar_t>(), \
k_extend.data_ptr<scalar_t>(), \
v_extend.data_ptr<scalar_t>(), \
k_buffer.data_ptr<scalar_t>(), \
v_buffer.data_ptr<scalar_t>(), \
req_to_token.data_ptr<index_t>(), \
req_pool_indices.data_ptr<int64_t>(), \
seq_lens.data_ptr<int64_t>(), \
encoder_lens_t.data_ptr<int64_t>(), \
extend_seq_lens.data_ptr<index_t>(), \
extend_start_loc.data_ptr<index_t>(), \
buffer.data_ptr(), \
sinks_tensor.data_ptr<scalar_t>(), \
tree_mask_ptr, \
num_seqs, \
num_heads, \
num_heads_kv, \
head_size, \
head_size_v, \
q_strideM, \
q_strideH, \
ke_strideN, \
ke_strideH, \
ve_strideN, \
ve_strideH, \
k_strideN, \
k_strideH, \
v_strideN, \
v_strideH, \
sm_scale, \
max_num_reqs, \
max_context_len, \
max_total_num_tokens, \
max_len_extend, \
sz, \
sliding_window_size, \
is_prefix_skipped, \
is_cross_attn, \
has_encoder_lens, \
has_sink); \
} while (0)
// q_extend, k_extend, v_extend, o_extend: contiguous tensors
// k_buffer, v_buffer: (prefix + extend) tensors in mem_manager
//
// q_extend: [num_tokens, num_heads, head_size]
// k_extend: [num_extend_tokens, num_heads, head_size]
// v_extend: [num_extend_tokens, num_heads, head_size]
// o_extend: [num_tokens, num_heads, head_size]
// k_buffer: [max_total_num_tokens, num_heads, head_size]
// v_buffer: [max_total_num_tokens, num_heads, head_size]
// req_to_token: [max_num_reqs, max_context_len] int32 or int64
// req_pool_indices: [num_seqs] int64
// seq_lens: [num_seqs] int64
// extend_seq_lens: [num_seqs]
// extend_start_loc: [num_seqs]
// encoder_lens: [num_seqs] int64 or None
// sinks: [num_heads] or None
// tree_mask: [num_seqs * max_len_extend * max_len_extend] bool or None
// TreeMaskMode::QLEN_ONLY tree mask for speculative TARGET_VERIFY; see [NOTE] 5 above.
void extend_attention_cpu(
at::Tensor& q_extend,
const std::optional<at::Tensor>& k_extend_opt,
const std::optional<at::Tensor>& v_extend_opt,
at::Tensor& o_extend,
at::Tensor& k_buffer,
at::Tensor& v_buffer,
at::Tensor& req_to_token,
at::Tensor& req_pool_indices,
at::Tensor& seq_lens,
at::Tensor& extend_seq_lens,
at::Tensor& extend_start_loc,
int64_t max_len_extend,
double sm_scale,
double logit_cap,
bool is_cross_attn,
int64_t sliding_window_size,
std::optional<at::Tensor> encoder_lens,
std::optional<at::Tensor> sinks,
std::optional<at::Tensor> tree_mask) {
if (!is_cross_attn) {
TORCH_CHECK(
k_extend_opt.has_value() && v_extend_opt.has_value(),
"k_extend and v_extend are required for non-cross attention");
}
// Since k_extend and v_extend are not used for cross attention, they can be initialized as k_buffer and v_buffer
// here.
auto k_extend = k_extend_opt.has_value() ? k_extend_opt.value() : k_buffer;
auto v_extend = v_extend_opt.has_value() ? v_extend_opt.value() : v_buffer;
CHECK_LAST_DIM_CONTIGUOUS_INPUT(q_extend);
CHECK_INPUT(o_extend);
CHECK_LAST_DIM_CONTIGUOUS_INPUT(k_extend);
CHECK_LAST_DIM_CONTIGUOUS_INPUT(v_extend);
CHECK_LAST_DIM_CONTIGUOUS_INPUT(k_buffer);
CHECK_LAST_DIM_CONTIGUOUS_INPUT(v_buffer);
int num_seqs = seq_lens.size(0);
int max_num_reqs = req_to_token.size(0);
int max_context_len = req_to_token.size(1);
int max_total_num_tokens = k_buffer.size(0);
int num_heads = q_extend.size(1);
int num_heads_kv = k_extend.size(1);
int head_size = q_extend.size(2);
int head_size_v = v_extend.size(2);
// strides for q_extend, k_extend and v_extend
int q_strideM = q_extend.stride(0);
int q_strideH = q_extend.stride(1);
int ke_strideN = k_extend.stride(0);
int ke_strideH = k_extend.stride(1);
int ve_strideN = v_extend.stride(0);
int ve_strideH = v_extend.stride(1);
// strides for k_buffer and v_buffer
int k_strideN = k_buffer.stride(0);
int k_strideH = k_buffer.stride(1);
int v_strideN = v_buffer.stride(0);
int v_strideH = v_buffer.stride(1);
// check sizes
CHECK_EQ(req_pool_indices.size(0), num_seqs);
CHECK_EQ(extend_seq_lens.size(0), num_seqs);
CHECK_EQ(extend_start_loc.size(0), num_seqs);
CHECK_EQ(v_extend.size(1), num_heads_kv);
CHECK_EQ(k_buffer.size(1), v_buffer.size(1));
// MLA will skip prefix part
const bool is_prefix_skipped = k_buffer.size(1) != num_heads_kv;
// check index data types
const auto index_dtype = req_to_token.scalar_type();
TORCH_CHECK(
index_dtype == at::kInt || index_dtype == at::kLong,
"extend: expect req_to_token to be int32 or int64, got ",
index_dtype);
TORCH_CHECK(seq_lens.scalar_type() == at::kLong, "extend: expect req_lens to be int64, got ", seq_lens.scalar_type());
TORCH_CHECK(
req_pool_indices.scalar_type() == at::kLong,
"extend: expect req_pool_indices to be int64, got ",
req_pool_indices.scalar_type());
TORCH_CHECK(
extend_seq_lens.scalar_type() == index_dtype && extend_start_loc.scalar_type() == index_dtype,
"extend: expect extend_seq_lens and extend_start_loc to have same dtype as req_to_token.");
// D and DV need to be 32x as we transpose by 512-bit
TORCH_CHECK(head_size % 32 == 0, "invalid head_size ", head_size);
TORCH_CHECK(head_size_v % 32 == 0, "invalid head_size_v ", head_size_v);
int num_threads = at::get_num_threads();
auto buffer = at::empty({}, q_extend.options().dtype(at::kChar));
bool has_encoder_lens = encoder_lens.has_value();
// Since encoder_lens is not used when it is None, encoder_lens_t can be initialized as any tensor of int64_t dtype.
at::Tensor encoder_lens_t = seq_lens;
if (has_encoder_lens) {
encoder_lens_t = encoder_lens.value();
CHECK_EQ(encoder_lens_t.size(0), num_seqs);
}
bool has_sink = sinks.has_value();
at::Tensor sinks_tensor = has_sink ? sinks.value() : at::empty({num_heads}, q_extend.options());
CHECK_DIM(1, sinks_tensor);
CHECK_EQ(sinks_tensor.size(0), num_heads);
const bool* tree_mask_ptr = nullptr;
if (tree_mask.has_value()) {
const at::Tensor& tree_mask_t = tree_mask.value();
CHECK_INPUT(tree_mask_t);
TORCH_CHECK(
tree_mask_t.scalar_type() == at::kBool, "extend: expect tree_mask to be bool, got ", tree_mask_t.scalar_type());
TORCH_CHECK(
tree_mask_t.numel() == static_cast<int64_t>(num_seqs) * max_len_extend * max_len_extend,
"extend: expect tree_mask numel to be num_seqs * max_len_extend^2 = ",
static_cast<int64_t>(num_seqs) * max_len_extend * max_len_extend,
", got ",
tree_mask_t.numel());
TORCH_CHECK(!is_cross_attn, "extend: tree_mask is not supported for cross attention");
// The window mask derives query positions from the row index
// (seq_len_prefix + m + row), but tree-mask rows sit at their tree depth,
// which is <= the row index; combining the two would over-mask the prefix.
TORCH_CHECK(sliding_window_size <= 0, "extend: tree_mask is not supported with sliding window attention");
tree_mask_ptr = tree_mask_t.data_ptr<bool>();
}
AT_DISPATCH_REDUCED_FLOATING_TYPES(q_extend.scalar_type(), "extend_attention_kernel", [&] {
AT_DISPATCH_INDEX_TYPES(index_dtype, "extend_attention_indices", [&] {
if (max_len_extend <= 256) {
LAUNCH_EXTEND_ATTENTION_KERNEL(32, 64);
} else if (max_len_extend <= 1024) {
LAUNCH_EXTEND_ATTENTION_KERNEL(128, 256);
} else if (max_len_extend <= 4096) {
LAUNCH_EXTEND_ATTENTION_KERNEL(256, 768);
} else { // max_len_extend > 4096
LAUNCH_EXTEND_ATTENTION_KERNEL(512, 768);
}
});
});
}
+250
View File
@@ -0,0 +1,250 @@
// Adapted from
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
// clang-format off
#pragma once
#include "common.h"
#include "vec.h"
#include "vec_pack.h"
template <typename scalar_t>
inline void fill_stub(scalar_t* __restrict__ out, float val, int size) {
using Vec = at::vec::Vectorized<scalar_t>;
constexpr int kVecSize = Vec::size();
const Vec data_vec = Vec(static_cast<scalar_t>(val));
int d = 0;
#pragma GCC unroll 4
for (; d <= size - kVecSize; d += kVecSize) {
data_vec.store(out + d);
}
if (size - d > 0) {
data_vec.store(out + d, size - d);
}
}
template <typename scalar_t, int BLOCK_N>
inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input) {
static_assert(BLOCK_N % 32 == 0);
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
constexpr int COLS = BLOCK_N / 16;
auto store = [&](auto i) {
constexpr int col = i % COLS;
// for COLS = 2, 4 use 512bit store
if constexpr (col % 2 == 0) {
auto [a_fvec0, a_fvec1] = load_float_vec2(input + col * 16);
bVec out_bvec = convert_from_float_ext<scalar_t>(a_fvec0, a_fvec1);
out_bvec.store(out + col * 16);
}
};
Unroll<COLS>{}(store);
}
template <typename scalar_t>
inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ acc, float s, int size) {
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
constexpr int kVecSize = bVec::size();
const fVec s_fvec = fVec(s);
int d = 0;
#pragma GCC unroll 4
for (; d <= size - kVecSize; d += kVecSize) {
auto [a_fvec0, a_fvec1] = load_float_vec2(acc + d);
a_fvec0 = a_fvec0 * s_fvec;
a_fvec1 = a_fvec1 * s_fvec;
bVec out_bvec = convert_from_float_ext<scalar_t>(a_fvec0, a_fvec1);
out_bvec.store(out + d);
}
for (; d < size; ++d) {
out[d] = static_cast<scalar_t>(acc[d] * s);
}
}
#if defined(CPU_CAPABILITY_AVX512)
template <>
inline void copy_stub<at::BFloat16>(at::BFloat16* __restrict__ out, const float* __restrict__ acc, float s, int size) {
const __m512 vscale = _mm512_set1_ps(s);
int d = 0;
#pragma GCC unroll 4
for (; d <= size - 32; d += 32) {
__m512 va0 = _mm512_mul_ps(_mm512_loadu_ps(acc + d), vscale);
__m512 va1 = _mm512_mul_ps(_mm512_loadu_ps(acc + d + 16), vscale);
__m512i vb = (__m512i)(_mm512_cvtne2ps_pbh(va1, va0));
_mm512_storeu_si512(out + d, vb);
}
int remainder = size - d;
if (remainder > 0) {
if (remainder <= 16) {
const __mmask16 vmask = (1ULL << remainder) - 1;
__m512 va = _mm512_mul_ps(_mm512_maskz_loadu_ps(vmask, acc + d), vscale);
__m256i vb = (__m256i)(_mm512_cvtneps_pbh(va));
_mm256_mask_storeu_epi16(reinterpret_cast<__m256i*>(out + d), vmask, vb);
} else { // remainder > 16
const __mmask16 vmask = (1ULL << (remainder - 16)) - 1;
__m512 va0 = _mm512_mul_ps(_mm512_loadu_ps(acc + d), vscale);
__m512 va1 = _mm512_mul_ps(_mm512_maskz_loadu_ps(vmask, acc + d + 16), vscale);
__m512i vb = (__m512i)(_mm512_cvtne2ps_pbh(va1, va0));
const __mmask32 vmask2 = (1ULL << remainder) - 1;
_mm512_mask_storeu_epi16(reinterpret_cast<__m512i*>(out + d), vmask2, vb);
}
}
}
#endif
template <typename scalar_t, int BLOCK_M, int BLOCK_N>
struct flash_attn_softmax {
static inline void apply(
float* __restrict__ s_i,
scalar_t* __restrict__ s_delta2,
float* __restrict__ v_prime,
float* __restrict__ s_prime,
float* __restrict__ m_prime,
int m_size,
int n_size,
int padded_n_size,
int head_size_v,
const float sm_scale,
int row) {
using Vec = at::vec::Vectorized<float>;
const Vec scale_vec = Vec(sm_scale);
float* s_delta = s_i;
// s_i <- s_i * scale
at::vec::map<float>([scale_vec](Vec x) { return x * scale_vec; }, s_i + row * BLOCK_N, s_i + row * BLOCK_N, n_size);
// m_i: max value per row
float m_i =
at::vec::reduce_all<float>([](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i + row * BLOCK_N, n_size);
m_i = std::max(m_i, m_prime[row]);
// m_delta <- exp(m' - m_i)
float m_delta = std::exp(m_prime[row] - m_i);
// s_delta <- exp(s_i - m_i)
at::vec::map<float>(
[m_i](Vec x) { return (x - Vec(m_i)).fexp_u20(); }, s_delta + row * BLOCK_N, s_i + row * BLOCK_N, n_size);
// s' <- s' * m_delta + sum(s_delta)
s_prime[row] *= m_delta;
s_prime[row] += at::vec::reduce_all<float>([](Vec& x, Vec& y) { return x + y; }, s_delta + row * BLOCK_N, n_size);
m_prime[row] = m_i;
// v' <- v' * m_delta
at::vec::map<float>(
[m_delta](Vec x) { return x * Vec(m_delta); },
v_prime + row * head_size_v,
v_prime + row * head_size_v,
head_size_v);
// Keep s_delta row-major for the following brgemm(P @ V), and only
// convert the columns that brgemm will consume.
fill_stub(s_delta + row * BLOCK_N + n_size, 0.f, padded_n_size - n_size);
copy_stub<scalar_t>(s_delta2 + row * BLOCK_N, s_delta + row * BLOCK_N, 1.f, padded_n_size);
}
};
#if defined(CPU_CAPABILITY_AVX512)
template <int BLOCK_M, int BLOCK_N>
struct flash_attn_softmax<at::BFloat16, BLOCK_M, BLOCK_N> {
static inline void apply(
float* __restrict__ s_i,
at::BFloat16* __restrict__ s_delta2,
float* __restrict__ v_prime,
float* __restrict__ s_prime,
float* __restrict__ m_prime,
int m_size,
int n_size,
int padded_n_size,
int head_size_v,
const float sm_scale,
int row) {
float* s_delta = s_i;
const __m512 vscale = _mm512_set1_ps(sm_scale);
int n_remainder = n_size & 15; // 0xF
const __mmask16 vmask = (1ULL << n_remainder) - 1;
int v_remainder = head_size_v & 15; // 0xF
const __mmask16 vmask1 = (1ULL << v_remainder) - 1;
constexpr float NEG_INF = -std::numeric_limits<float>::infinity();
__m512 va;
__m256i vb;
__m512 vmax;
__m512 vsum;
__m512 vmdelta;
const __m512 vneg_inf = _mm512_set1_ps(NEG_INF);
int m = row;
vmax = vneg_inf;
// s_i <- s_i * scale
int n = 0;
for (; n <= n_size - 16; n += 16) {
va = _mm512_mul_ps(_mm512_loadu_ps(s_i + m * BLOCK_N + n), vscale);
vmax = _mm512_max_ps(va, vmax);
}
if (n_remainder > 0) {
va = _mm512_mul_ps(_mm512_mask_loadu_ps(vneg_inf, vmask, s_i + m * BLOCK_N + n), vscale);
vmax = _mm512_max_ps(va, vmax);
}
// m_i: max value per row
float m_i = _mm512_reduce_max_ps(vmax);
m_i = std::max(m_i, m_prime[m]);
vmax = _mm512_set1_ps(m_i);
// m_delta <- exp(m' - m_i)
float m_delta = std::exp(m_prime[m] - m_i);
// s_delta <- exp(s_i - m_i)
vsum = _mm512_setzero_ps();
for (n = 0; n <= n_size - 16; n += 16) {
va = _mm512_mul_ps(_mm512_loadu_ps(s_i + m * BLOCK_N + n), vscale);
va = _mm512_fexp_u20_ps(_mm512_sub_ps(va, vmax));
vsum = _mm512_add_ps(vsum, va);
vb = (__m256i)(_mm512_cvtneps_pbh(va));
_mm256_storeu_si256(reinterpret_cast<__m256i*>(s_delta2 + m * BLOCK_N + n), vb);
}
if (n_remainder > 0) {
va = _mm512_mul_ps(_mm512_mask_loadu_ps(vneg_inf, vmask, s_i + m * BLOCK_N + n), vscale);
va = _mm512_fexp_u20_ps(_mm512_sub_ps(va, vmax));
vsum = _mm512_add_ps(vsum, va);
vb = (__m256i)(_mm512_cvtneps_pbh(va));
_mm256_mask_storeu_epi16(reinterpret_cast<__m256i*>(s_delta2 + m * BLOCK_N + n), vmask, vb);
}
// s' <- s' * m_delta + sum(s_delta)
s_prime[m] *= m_delta;
s_prime[m] += _mm512_reduce_add_ps(vsum);
m_prime[m] = m_i;
// pad s_delta with 0, pad_size range from [0, 32)
int pad_size = padded_n_size - n_size;
if (pad_size > 0) {
const __m512i vzero = _mm512_setzero_si512();
__mmask32 vmask2 = (1ULL << pad_size) - 1;
_mm512_mask_storeu_epi16(reinterpret_cast<__m512i*>(s_delta2 + m * BLOCK_N + n_size), vmask2, vzero);
}
// v' <- v' * m_delta
vmdelta = _mm512_set1_ps(m_delta);
int k = 0;
for (; k <= head_size_v - 16; k += 16) {
va = _mm512_mul_ps(_mm512_loadu_ps(v_prime + m * head_size_v + k), vmdelta);
_mm512_storeu_ps(reinterpret_cast<__m512*>(v_prime + m * head_size_v + k), va);
}
if (v_remainder > 0) {
va = _mm512_mul_ps(_mm512_maskz_loadu_ps(vmask1, v_prime + m * head_size_v + k), vmdelta);
_mm512_mask_storeu_ps(reinterpret_cast<__m512*>(v_prime + m * head_size_v + k), vmask1, va);
}
}
};
#endif
+111
View File
@@ -0,0 +1,111 @@
// vLLM-native CPU cache-write op for MLA's single-latent-buffer KV cache.
//
// `concat_and_cache_mla` (the generic MLA cache-write op used by every GPU
// backend) is registered CUDA-only. This is the CPU counterpart, adapted in
// spirit from SGLang's `store_cache_cpu` (csrc/cpu/kvcache.cpp) but
// generalized to write two source tensors (`kv_c_normed`, `k_pe`) into two
// different column-offset ranges of the SAME destination row -- SGLang's
// version assumes k/v land in two independent, equal-row-width cache
// tensors, which doesn't hold here since MLA's cache is one 576-wide buffer
// and the two column ranges (512-wide, 64-wide) don't match the buffer's
// true per-token stride, so the write can't reuse `store_cache_cpu` as-is.
#include "common.h"
#include "vec.h"
namespace {
template <typename scalar_t>
inline void copy_stub(scalar_t* __restrict__ dst,
const scalar_t* __restrict__ src, int64_t size) {
int64_t d = 0;
#if defined(CPU_CAPABILITY_AVX512)
using Vec = at::vec::Vectorized<scalar_t>;
constexpr int64_t kVecSize = Vec::size();
for (; d <= size - kVecSize; d += kVecSize) {
Vec data = Vec::loadu(src + d);
data.store(dst + d);
}
#endif
for (; d < size; ++d) {
dst[d] = src[d];
}
}
template <typename scalar_t, typename index_t>
void concat_and_cache_mla_kernel_impl(
const scalar_t* __restrict__ kv_c_normed, // [num_tokens, kv_lora_rank]
const scalar_t* __restrict__ k_pe, // [num_tokens, qk_rope_head_dim]
scalar_t* __restrict__ kv_cache, // [.., kv_lora_rank + qk_rope_head_dim]
const index_t* __restrict__ slot_mapping, // [num_tokens]
int64_t num_tokens, int64_t kv_lora_rank, int64_t qk_rope_head_dim,
int64_t kv_c_stride, int64_t k_pe_stride, int64_t cache_stride) {
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
for (int64_t i = begin; i < end; ++i) {
int64_t slot = static_cast<int64_t>(slot_mapping[i]);
if (slot < 0) {
// padded/invalid token, matches `concat_and_cache_mla`'s semantics.
continue;
}
scalar_t* __restrict__ cache_row = kv_cache + slot * cache_stride;
copy_stub(cache_row, kv_c_normed + i * kv_c_stride, kv_lora_rank);
copy_stub(cache_row + kv_lora_rank, k_pe + i * k_pe_stride,
qk_rope_head_dim);
}
});
}
} // namespace
// kv_c_normed : [num_tokens, kv_lora_rank]
// k_pe : [num_tokens, qk_rope_head_dim] or [num_tokens, 1,
// qk_rope_head_dim] kv_cache : [num_blocks, block_size, kv_lora_rank +
// qk_rope_head_dim] slot_mapping: [num_tokens] int32/int64, absolute physical
// row index
// (block_id * block_size + block_offset); negative entries are
// skipped (padded tokens), matching `concat_and_cache_mla`.
void concat_and_cache_mla_cpu(const at::Tensor& kv_c_normed,
const at::Tensor& k_pe, at::Tensor& kv_cache,
const at::Tensor& slot_mapping) {
TORCH_CHECK(kv_c_normed.dim() == 2,
"kv_c_normed must be 2D [num_tokens, kv_lora_rank]");
TORCH_CHECK(k_pe.dim() == 2 || k_pe.dim() == 3, "k_pe must be 2D or 3D");
TORCH_CHECK(kv_cache.dim() == 3,
"kv_cache must be 3D [num_blocks, block_size, head_size]");
TORCH_CHECK(kv_c_normed.stride(-1) == 1,
"kv_c_normed innermost dim must be contiguous");
TORCH_CHECK(k_pe.stride(-1) == 1, "k_pe innermost dim must be contiguous");
TORCH_CHECK(kv_cache.stride(-1) == 1,
"kv_cache innermost dim must be contiguous");
int64_t num_tokens = kv_c_normed.size(0);
int64_t kv_lora_rank = kv_c_normed.size(1);
int64_t qk_rope_head_dim = k_pe.size(-1);
int64_t head_size = kv_cache.size(-1);
TORCH_CHECK(head_size == kv_lora_rank + qk_rope_head_dim,
"kv_cache head_size must equal kv_lora_rank + qk_rope_head_dim");
TORCH_CHECK(slot_mapping.size(0) == num_tokens, "slot_mapping size mismatch");
// Real physical per-token stride of the cache, read from the tensor's own
// strides rather than assumed to equal head_size (paged/pooled buffers are
// not guaranteed to have zero inter-row padding in general, even though in
// practice a freshly-allocated MLA cache is fully contiguous).
int64_t cache_stride = kv_cache.stride(1);
const auto dtype = kv_cache.scalar_type();
TORCH_CHECK(dtype == kv_c_normed.scalar_type() && dtype == k_pe.scalar_type(),
"concat_and_cache_mla_cpu: dtype mismatch");
const auto index_dtype = slot_mapping.scalar_type();
TORCH_CHECK(index_dtype == at::kLong || index_dtype == at::kInt,
"slot_mapping must be int32 or int64");
AT_DISPATCH_REDUCED_FLOATING_TYPES(dtype, "concat_and_cache_mla_cpu", [&] {
AT_DISPATCH_INDEX_TYPES(index_dtype, "concat_and_cache_mla_cpu_index", [&] {
concat_and_cache_mla_kernel_impl<scalar_t, index_t>(
kv_c_normed.data_ptr<scalar_t>(), k_pe.data_ptr<scalar_t>(),
kv_cache.data_ptr<scalar_t>(), slot_mapping.data_ptr<index_t>(),
num_tokens, kv_lora_rank, qk_rope_head_dim, kv_c_normed.stride(0),
k_pe.stride(0), cache_stride);
});
});
}
+62
View File
@@ -91,6 +91,39 @@ at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2,
const std::optional<at::Tensor>& bias,
at::ScalarType out_dtype, bool is_vnni);
// Adapted from sglang: MLA CPU kernels (AMX-only)
void decode_attention_cpu(at::Tensor& query, at::Tensor& k_buffer,
at::Tensor& v_buffer, at::Tensor& output,
const std::optional<at::Tensor>& key,
const std::optional<at::Tensor>& value,
const std::optional<at::Tensor>& loc,
at::Tensor& attn_logits, at::Tensor& req_to_token,
at::Tensor& req_pool_indices, at::Tensor& seq_lens,
double sm_scale, double logit_cap, bool is_cross_attn,
int64_t sliding_window_size,
std::optional<at::Tensor> encoder_lens,
std::optional<at::Tensor> sinks);
void extend_attention_cpu(
at::Tensor& q_extend, const std::optional<at::Tensor>& k_extend,
const std::optional<at::Tensor>& v_extend, at::Tensor& o_extend,
at::Tensor& k_buffer, at::Tensor& v_buffer, at::Tensor& req_to_token,
at::Tensor& req_pool_indices, at::Tensor& seq_lens,
at::Tensor& extend_seq_lens, at::Tensor& extend_start_loc,
int64_t max_len_extend, double sm_scale, double logit_cap,
bool is_cross_attn, int64_t sliding_window_size,
std::optional<at::Tensor> encoder_lens, std::optional<at::Tensor> sinks,
std::optional<at::Tensor> tree_mask);
void bmm_cpu(at::Tensor& out, at::Tensor& mat1, at::Tensor& mat2, bool is_vnni,
const std::optional<at::Tensor>& scale);
// vLLM-native: CPU cache-write op for MLA's single-latent-buffer KV cache
// (the CUDA-only `concat_and_cache_mla` has no CPU dispatch).
void concat_and_cache_mla_cpu(const at::Tensor& kv_c_normed,
const at::Tensor& k_pe, at::Tensor& kv_cache,
const at::Tensor& slot_mapping);
// Adapted from sglang: INT4 W4A8 kernels
std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_weight_packed_scale_zp(
at::Tensor qweight, // awq: (*, K, N / 8) || gptq: (*, K / 8, N) , int32
@@ -521,6 +554,35 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"pad_slot_id, "
"bool is_vnni) -> Tensor");
ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu);
// Adapted from sglang: MLA CPU kernels (AMX-only, DeepSeek V2/V3/R1)
ops.def(
"decode_attention_cpu(Tensor query, Tensor k_buffer, Tensor v_buffer, "
"Tensor(a!) output, Tensor? key, Tensor? value, Tensor? loc, Tensor "
"attn_logits, Tensor req_to_token, Tensor req_pool_indices, Tensor "
"seq_lens, float sm_scale, float logit_cap, bool is_cross_attn, int "
"sliding_window_size, Tensor? encoder_lens, Tensor? sinks) -> ()");
ops.impl("decode_attention_cpu", torch::kCPU, &decode_attention_cpu);
ops.def(
"extend_attention_cpu(Tensor q_extend, Tensor? k_extend, Tensor? "
"v_extend, Tensor(a!) o_extend, Tensor k_buffer, Tensor v_buffer, "
"Tensor req_to_token, Tensor req_pool_indices, Tensor seq_lens, Tensor "
"extend_seq_lens, Tensor extend_start_loc, int max_len_extend, float "
"sm_scale, float logit_cap, bool is_cross_attn, int "
"sliding_window_size, Tensor? encoder_lens, Tensor? sinks, Tensor? "
"tree_mask=None) -> ()");
ops.impl("extend_attention_cpu", torch::kCPU, &extend_attention_cpu);
ops.def(
"bmm_cpu(Tensor(a!) out, Tensor mat1, Tensor mat2, bool is_vnni, "
"Tensor? scale) -> ()");
ops.impl("bmm_cpu", torch::kCPU, &bmm_cpu);
ops.def(
"concat_and_cache_mla_cpu(Tensor kv_c_normed, Tensor k_pe, "
"Tensor(a!) kv_cache, Tensor slot_mapping) -> ()");
ops.impl("concat_and_cache_mla_cpu", torch::kCPU, &concat_and_cache_mla_cpu);
#endif
#if (defined(__AVX512BF16__) && defined(__AVX512F__) && \
+471
View File
@@ -0,0 +1,471 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the AMX-only CPU MLA backend: the vendored
decode/extend/bmm kernels, the KV cache write, and the ``AMXMLAImpl``
backend built on top of them.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
if not torch.cpu._is_amx_tile_supported():
pytest.skip("AMX MLA requires an AMX-capable host", allow_module_level=True)
torch.cpu._init_amx()
from vllm import _custom_ops as ops # noqa: E402
from vllm.utils.torch_utils import set_random_seed # noqa: E402
from vllm.v1.attention.backends.mla.amx_mla import ( # noqa: E402
AMXMLAImpl,
_compute_num_kv_splits,
_expand_block_table,
)
KV_LORA_RANK = 64
QK_ROPE_HEAD_DIM = 32
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM # 96, kv-cache row width
# The extend/decode kernels transpose by 512-bit lanes: both the cache row
# width and the (kv_lora_rank-wide) value width must be 32-element aligned.
DTYPE = torch.bfloat16
ATOL = 2e-2
RTOL = 2e-2
# Backend-level tests chain absorb -> attend -> de-absorb through bf16,
# accumulating more rounding error than a single kernel op.
_IMPL_ATOL = 1.5e-1
_IMPL_RTOL = 1.5e-1
def _flatten_cache(kv_cache: torch.Tensor) -> torch.Tensor:
"""(num_blocks, block_size, head_size) -> (num_slots, head_size)."""
return kv_cache.view(-1, kv_cache.size(-1))
def _random_paged_cache(num_slots: int, head_size: int) -> torch.Tensor:
return torch.randn(num_slots, head_size, dtype=DTYPE)
def _ref_latent_attn(
q: torch.Tensor,
keys: torch.Tensor,
scale: float,
kv_lora_rank: int,
) -> torch.Tensor:
"""Plain-PyTorch causal-free latent-space MQA attention for one request.
q: (num_heads, head_size); keys: (ctx_len, head_size). Values are the
first ``kv_lora_rank`` columns of the same latent rows (MLA's K/V
aliasing), matching what decode_attention_cpu/extend_attention_cpu
compute internally for the MLA-shaped case.
"""
values = keys[:, :kv_lora_rank]
scores = (q.float() @ keys.float().T) * scale
probs = torch.softmax(scores, dim=-1)
return (probs @ values.float()).to(q.dtype)
def test_bmm_cpu_matches_torch_bmm():
set_random_seed(0)
# bmm_cpu requires the output's last dim (mat2's out-features) to be a
# multiple of 32 (tinygemm's tile width); the contraction dim is free.
n, b, p, l = 8, 32, 32, 32 # noqa: E741
mat1 = torch.randn(n, b, p, dtype=DTYPE)
mat2 = torch.randn(n, l, p, dtype=DTYPE) # (N, OUT, IN) Linear convention
ref = torch.bmm(mat1.float(), mat2.float().transpose(1, 2)).to(DTYPE)
out = torch.empty(n, b, l, dtype=DTYPE)
ops.bmm_cpu(out, mat1, mat2, False, None)
torch.testing.assert_close(out, ref, atol=ATOL, rtol=RTOL)
packed = torch.ops._C.convert_weight_packed(mat2.contiguous())
out_vnni = torch.empty(n, b, l, dtype=DTYPE)
ops.bmm_cpu(out_vnni, mat1, packed, True, None)
torch.testing.assert_close(out_vnni, ref, atol=ATOL, rtol=RTOL)
def test_concat_and_cache_mla_round_trip():
set_random_seed(0)
num_tokens = 37
num_blocks, block_size = 8, 16
kv_cache = torch.zeros(num_blocks, block_size, HEAD_SIZE, dtype=DTYPE)
kv_c_normed = torch.randn(num_tokens, KV_LORA_RANK, dtype=DTYPE)
k_pe = torch.randn(num_tokens, QK_ROPE_HEAD_DIM, dtype=DTYPE)
slot_mapping = torch.randperm(num_blocks * block_size)[:num_tokens].to(torch.int64)
ops.amx_mla_concat_and_cache(kv_c_normed, k_pe, kv_cache, slot_mapping)
flat = _flatten_cache(kv_cache)
read_back = flat[slot_mapping]
expected = torch.cat([kv_c_normed, k_pe], dim=-1)
torch.testing.assert_close(read_back, expected)
untouched = torch.ones(num_blocks * block_size, dtype=torch.bool)
untouched[slot_mapping] = False
assert (flat[untouched] == 0).all()
@pytest.mark.parametrize("seq_lens", [[5], [1, 300, 33], [513, 1, 129, 7]])
def test_decode_attention_cpu_matches_reference(seq_lens):
set_random_seed(0)
num_seqs = len(seq_lens)
num_heads = 4
block_size = 32
max_len = max(seq_lens)
max_blocks = (max_len + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
scale = HEAD_SIZE**-0.5
q = torch.randn(num_seqs, num_heads, HEAD_SIZE, dtype=DTYPE)
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
o = torch.zeros(num_seqs, num_heads, KV_LORA_RANK, dtype=DTYPE)
attn_logits = torch.zeros(
num_seqs, num_heads, 4, KV_LORA_RANK + 1, dtype=torch.float32
)
ops.cpu_mla_decode(
q,
kv_cache_flat,
v_buffer,
o,
None,
None,
None,
attn_logits,
req_to_token,
req_pool_indices,
seq_lens_t,
scale,
0.0,
False,
0,
None,
None,
)
flat_cache = _flatten_cache(kv_cache)
for i, seq_len in enumerate(seq_lens):
token_ids = req_to_token[i, :seq_len]
keys = flat_cache[token_ids]
ref = _ref_latent_attn(q[i], keys, scale, KV_LORA_RANK)
torch.testing.assert_close(o[i], ref, atol=ATOL, rtol=RTOL)
@pytest.mark.parametrize(
"prefix_lens,extend_lens",
[
([0], [9]), # fresh prefill, no cached prefix
([64], [17]), # continuation of a cached prefix
([0, 40, 128], [23, 5, 31]), # mixed batch
],
)
def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
set_random_seed(0)
num_seqs = len(prefix_lens)
num_heads = 4
block_size = 32
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
max_len = max(seq_lens)
max_blocks = (max_len + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
extend_seq_lens_t = torch.tensor(extend_lens, dtype=torch.int64)
extend_start_loc = torch.cumsum(
torch.tensor([0, *extend_lens[:-1]], dtype=torch.int64), dim=0
)
scale = HEAD_SIZE**-0.5
total_new_tokens = sum(extend_lens)
# The new tokens' latent K/V, already written into the cache (mirrors
# do_kv_cache_update running before forward_mha in real usage).
k_extend_flat = torch.randn(total_new_tokens, HEAD_SIZE, dtype=DTYPE)
flat_cache = _flatten_cache(kv_cache)
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
start = extend_start_loc[i].item()
flat_cache[new_token_ids] = k_extend_flat[start : start + ext_len]
q_extend = torch.randn(total_new_tokens, num_heads, HEAD_SIZE, dtype=DTYPE)
k_extend = k_extend_flat.unsqueeze(1)
v_extend = k_extend[..., :KV_LORA_RANK]
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
o_extend = torch.empty(total_new_tokens, num_heads, KV_LORA_RANK, dtype=DTYPE)
ops.cpu_mla_extend(
q_extend,
k_extend,
v_extend,
o_extend,
kv_cache_flat,
v_buffer,
req_to_token,
req_pool_indices,
seq_lens_t,
extend_seq_lens_t,
extend_start_loc,
max(extend_lens),
scale,
0.0,
False,
0,
None,
None,
None,
)
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
start = extend_start_loc[i].item()
for t in range(ext_len):
visible = prefix_len + t + 1
token_ids = req_to_token[i, :visible]
keys = flat_cache[token_ids]
ref = _ref_latent_attn(q_extend[start + t], keys, scale, KV_LORA_RANK)
torch.testing.assert_close(o_extend[start + t], ref, atol=ATOL, rtol=RTOL)
class _FakeLinear:
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
.quant_method are read by get_and_maybe_dequant_weights()."""
def __init__(self, weight: torch.Tensor):
self.weight = weight
self.quant_method = None
def _make_amx_mla_impl(num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank):
weight = torch.randn(
num_heads * (qk_nope_head_dim + v_head_dim), kv_lora_rank, dtype=DTYPE
)
kv_b_proj = _FakeLinear(weight)
impl = AMXMLAImpl(
num_heads=num_heads,
head_size=kv_lora_rank + QK_ROPE_HEAD_DIM,
scale=(qk_nope_head_dim + QK_ROPE_HEAD_DIM) ** -0.5,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="auto",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
qk_head_dim=qk_nope_head_dim + QK_ROPE_HEAD_DIM,
v_head_dim=v_head_dim,
kv_b_proj=kv_b_proj,
)
impl.process_weights_after_loading(DTYPE)
return impl, weight
class _FakePrefillMetadata:
def __init__(
self, block_table, cpu_seq_lens, query_start_loc, req_to_token, req_pool_indices
):
self.block_table = block_table
self.cpu_seq_lens = cpu_seq_lens
query_start_loc_i64 = query_start_loc.to(torch.int64)
extend_seq_lens = query_start_loc_i64[1:] - query_start_loc_i64[:-1]
self.extend_seq_lens = extend_seq_lens
self.extend_start_loc = query_start_loc_i64[:-1]
self.max_len_extend = int(extend_seq_lens.max().item())
self.req_to_token = req_to_token
self.req_pool_indices = req_pool_indices
class _FakeDecodeMetadata:
def __init__(self, block_table, seq_lens, req_to_token, req_pool_indices):
self.block_table = block_table
self.seq_lens_i64 = seq_lens.to(torch.int64)
self.req_to_token = req_to_token
self.req_pool_indices = req_pool_indices
self.num_kv_splits = _compute_num_kv_splits(
int(seq_lens.max().item()), current_platform.num_compute_units()
)
class _FakeAttnMetadata:
def __init__(self, decode=None, prefill=None, max_seq_len=0):
self.decode = decode
self.prefill = prefill
self.max_seq_len = max_seq_len
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
decode attention through W_UK/W_UV."""
set_random_seed(1)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
)
w_uk, w_uv = kv_b_weight.T.view(
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
).split([qk_nope_head_dim, v_head_dim], dim=-1)
seq_lens = [1, 40, 5]
num_seqs = len(seq_lens)
block_size = 32
max_blocks = (max(seq_lens) + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
q_nope = torch.randn(num_seqs, num_heads, qk_nope_head_dim, dtype=DTYPE)
q_pe = torch.randn(num_seqs, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
# Pre-absorb, mirroring MLAAttention.forward_impl's own bmm against W_UK_T.
ql_nope = torch.einsum("bnp,lnp->bnl", q_nope.float(), w_uk.float()).to(DTYPE)
attn_metadata = _FakeAttnMetadata(
decode=_FakeDecodeMetadata(
block_table=block_table,
seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
req_to_token=req_to_token,
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
),
max_seq_len=max(seq_lens),
)
o, lse = impl.forward_mqa((ql_nope, q_pe), kv_cache, attn_metadata, layer=None)
assert lse is None
# De-absorb, mirroring MLAAttention.forward_impl's own bmm against
# layer._v_up_proj, to get back into real per-head V space for
# comparison against a from-scratch reference.
out_real = torch.einsum("bnl,lnv->bnv", o.float(), w_uv.float())
flat_cache = _flatten_cache(kv_cache)
scale = impl.scale
for i, seq_len in enumerate(seq_lens):
token_ids = req_to_token[i, :seq_len]
keys_latent = flat_cache[token_ids] # (L, head_size)
k_nope_latent, k_pe = keys_latent.split(
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
)
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
k_pe_b = k_pe.float().unsqueeze(0).expand(num_heads, -1, -1)
k_full = torch.cat([k_real, k_pe_b], dim=-1) # (num_heads, L, qk_head_dim)
q_full = torch.cat(
[q_nope[i], q_pe[i]], dim=-1
).float() # (num_heads, qk_head_dim)
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
probs = torch.softmax(scores, dim=-1)
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
ref = torch.einsum("nl,nlv->nv", probs, v_real)
torch.testing.assert_close(out_real[i], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL)
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
prefill batch with a mix of fresh and continued (cached-prefix)
sequences."""
set_random_seed(2)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
)
w_uk, w_uv = kv_b_weight.T.view(
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
).split([qk_nope_head_dim, v_head_dim], dim=-1)
prefix_lens = [0, 20]
extend_lens = [13, 7]
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
num_seqs = len(prefix_lens)
block_size = 32
max_blocks = (max(seq_lens) + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
flat_cache = _flatten_cache(kv_cache)
total_new_tokens = sum(extend_lens)
query_start_loc = torch.tensor(
[0, *torch.cumsum(torch.tensor(extend_lens), dim=0).tolist()],
dtype=torch.int64,
)
q_nope = torch.randn(total_new_tokens, num_heads, qk_nope_head_dim, dtype=DTYPE)
q_pe = torch.randn(total_new_tokens, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
q = torch.cat([q_nope, q_pe], dim=-1)
kv_c_normed = torch.randn(total_new_tokens, kv_lora_rank, dtype=DTYPE)
k_pe = torch.randn(total_new_tokens, 1, QK_ROPE_HEAD_DIM, dtype=DTYPE)
# Write the new tokens' latent K/V into the cache ahead of time, mirroring
# do_kv_cache_update running before forward_mha in real usage.
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
start = query_start_loc[i].item()
flat_cache[new_token_ids, :kv_lora_rank] = kv_c_normed[start : start + ext_len]
flat_cache[new_token_ids, kv_lora_rank:] = k_pe[start : start + ext_len, 0]
attn_metadata = _FakeAttnMetadata(
prefill=_FakePrefillMetadata(
block_table=block_table,
cpu_seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
query_start_loc=query_start_loc,
req_to_token=req_to_token,
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
)
)
output = torch.empty(total_new_tokens, num_heads * v_head_dim, dtype=DTYPE)
impl.forward_mha(
q, kv_c_normed, k_pe, kv_cache, attn_metadata, k_scale=None, output=output
)
out_view = output.view(total_new_tokens, num_heads, v_head_dim).float()
scale = impl.scale
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
start = query_start_loc[i].item()
for t in range(ext_len):
visible = prefix_len + t + 1
token_ids = req_to_token[i, :visible]
keys_latent = flat_cache[token_ids]
k_nope_latent, k_pe_ref = keys_latent.split(
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
)
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
k_pe_b = k_pe_ref.float().unsqueeze(0).expand(num_heads, -1, -1)
k_full = torch.cat([k_real, k_pe_b], dim=-1)
q_full = torch.cat([q_nope[start + t], q_pe[start + t]], dim=-1).float()
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
probs = torch.softmax(scores, dim=-1)
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
ref = torch.einsum("nl,nlv->nv", probs, v_real)
torch.testing.assert_close(
out_view[start + t], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL
)
+2
View File
@@ -172,9 +172,11 @@ def test_mla_post_load_preserves_runtime_weight_addresses(monkeypatch):
layer.kv_b_proj.quant_method = None
layer.is_aiter_triton_fp4_bmm_enabled = False
layer.is_aiter_triton_fp8_bmm_enabled = False
layer.is_amx_bmm_enabled = False
layer.dcp_q_replicate = False
layer.quant_config = None
layer.layer_name = "test"
layer.impl = SimpleNamespace(process_weights_after_loading=lambda act_dtype: None)
monkeypatch.setattr(
mla_attention_module, "set_default_quant_scales", lambda *_, **__: None
@@ -63,6 +63,7 @@ class TestGetMLAPrefillBackend:
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.get_device_capability.return_value = None
mock_platform.is_cpu.return_value = False
backend = get_mla_prefill_backend(vllm_config)
assert backend.get_name() == "FLASH_ATTN"
+103
View File
@@ -3937,6 +3937,109 @@ def cpu_attention_with_kv_cache(
)
def cpu_mla_decode(
query: torch.Tensor,
k_buffer: torch.Tensor,
v_buffer: torch.Tensor,
output: torch.Tensor,
key: torch.Tensor | None,
value: torch.Tensor | None,
loc: torch.Tensor | None,
attn_logits: torch.Tensor,
req_to_token: torch.Tensor,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
sm_scale: float,
logit_cap: float,
is_cross_attn: bool,
sliding_window_size: int,
encoder_lens: torch.Tensor | None,
sinks: torch.Tensor | None,
) -> None:
torch.ops._C.decode_attention_cpu(
query,
k_buffer,
v_buffer,
output,
key,
value,
loc,
attn_logits,
req_to_token,
req_pool_indices,
seq_lens,
sm_scale,
logit_cap,
is_cross_attn,
sliding_window_size,
encoder_lens,
sinks,
)
def cpu_mla_extend(
q_extend: torch.Tensor,
k_extend: torch.Tensor | None,
v_extend: torch.Tensor | None,
o_extend: torch.Tensor,
k_buffer: torch.Tensor,
v_buffer: torch.Tensor,
req_to_token: torch.Tensor,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
extend_seq_lens: torch.Tensor,
extend_start_loc: torch.Tensor,
max_len_extend: int,
sm_scale: float,
logit_cap: float,
is_cross_attn: bool,
sliding_window_size: int,
encoder_lens: torch.Tensor | None,
sinks: torch.Tensor | None,
tree_mask: torch.Tensor | None = None,
) -> None:
torch.ops._C.extend_attention_cpu(
q_extend,
k_extend,
v_extend,
o_extend,
k_buffer,
v_buffer,
req_to_token,
req_pool_indices,
seq_lens,
extend_seq_lens,
extend_start_loc,
max_len_extend,
sm_scale,
logit_cap,
is_cross_attn,
sliding_window_size,
encoder_lens,
sinks,
tree_mask,
)
def bmm_cpu(
out: torch.Tensor,
mat1: torch.Tensor,
mat2: torch.Tensor,
is_vnni: bool,
scale: torch.Tensor | None = None,
) -> None:
torch.ops._C.bmm_cpu(out, mat1, mat2, is_vnni, scale)
def amx_mla_concat_and_cache(
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
) -> None:
torch.ops._C.concat_and_cache_mla_cpu(kv_c_normed, k_pe, kv_cache, slot_mapping)
def cpu_gemm_wna16(
input: torch.Tensor,
q_weight: torch.Tensor,
@@ -540,6 +540,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
**extra_impl_args,
)
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
self.is_amx_bmm_enabled = getattr(self.impl, "uses_amx_bmm", False)
# AMX reads kv_b_proj's weight directly and never calls it live; the
# reference CPU MLA backend calls it but isn't perf-critical. Skip
# the packed-kernel dispatch either way.
kv_b_proj._cpu_skip_gemm_dispatch = True
self.use_direct_call = not current_platform.opaque_attention_op()
vllm_config = get_current_vllm_config()
@@ -908,6 +913,20 @@ class MLAAttention(nn.Module, AttentionLayerBase):
group_size=128,
transpose_bm=True,
)
elif self.is_amx_bmm_enabled:
# bmm_cpu computes out[n] = mat1[n] @ mat2[n]^T against
# AMXMLAImpl's own (N, L, P) packed W_UK -- same as prefill.
N, B, P = mqa_q_nope.shape
L = self.kv_lora_rank
mqa_ql_nope = mqa_q_nope.new_empty((N, B, L))
ops.bmm_cpu(
mqa_ql_nope,
mqa_q_nope,
self.impl._w_uk_packed, # type: ignore[attr-defined]
True,
None,
)
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
else:
# Pads the head_dim if necessary (for the underlying kernel)
N, B, P = mqa_q_nope.shape
@@ -1026,6 +1045,18 @@ class MLAAttention(nn.Module, AttentionLayerBase):
return output_padded
def process_weights_after_loading(self, act_dtype: torch.dtype):
# Let per-backend impls do their own weight packing first (no-op
# unless overridden), mirroring Attention.process_weights_after_loading.
self.impl.process_weights_after_loading(act_dtype)
if self.is_amx_bmm_enabled:
# AMXMLAImpl already packed its own W_UK/W_UV above, for both
# prefill and decode. Release the now-unused raw weight.
self.kv_b_proj.weight = torch.nn.Parameter(
torch.empty(0), requires_grad=False
)
return
# we currently do not have quantized bmm's which are needed for
# `W_UV` and `W_UK_T`, we just store fp16/bf16 copies and perform
# the bmm's in 16-bit, the extra memory overhead of this is fairly low
@@ -1192,6 +1223,16 @@ class MLAAttention(nn.Module, AttentionLayerBase):
x = rocm_aiter_ops.triton_fp8_bmm(
x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True, YQ=out
)
elif self.is_amx_bmm_enabled:
# bmm_cpu computes out[n] = mat1[n] @ mat2[n]^T against
# AMXMLAImpl's own (N, V, L) packed W_UV -- same as prefill.
ops.bmm_cpu(
out.transpose(0, 1),
x,
self.impl._w_uv_packed, # type: ignore[attr-defined]
True,
None,
)
else:
# Multiply + Transpose (N, B, L) x (N, L, V)->(N, B, V)->(B, N, V)
torch.bmm(x, self.W_UV, out=out.transpose(0, 1))
+6
View File
@@ -197,6 +197,12 @@ class UnquantizedLinearMethod(LinearMethodBase):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if current_platform.is_cpu():
# MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): not
# perf-critical, so skip packing and use a plain fallback.
if getattr(layer, "_cpu_skip_gemm_dispatch", False):
layer.cpu_linear = torch.nn.functional.linear
return
from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm
dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
+31 -5
View File
@@ -89,13 +89,30 @@ class CpuPlatform(Platform):
if attn_selector_config.use_sparse:
raise NotImplementedError("Sparse Attention is not supported on CPU.")
if attn_selector_config.use_mla:
amx_available = (
cls.get_cpu_architecture() == CpuArchEnum.X86
and torch.cpu._is_amx_tile_supported()
)
if amx_available and selected_backend != AttentionBackendEnum.CPU_MLA:
# Prefer AMX when available, unless CPU_MLA was explicitly requested.
if (
selected_backend
and selected_backend != AttentionBackendEnum.AMX_MLA
):
logger.info("Cannot use %s backend on CPU.", selected_backend)
logger.info_once("Using %s backend.", AttentionBackendEnum.AMX_MLA.name)
return AttentionBackendEnum.AMX_MLA.get_path()
# Reference MLA implementation on CPU. Performance is not the
# goal here; the backend simply wires the CPU decode kernel
# (`mla_decode_kvcache`) and an SDPA-based prefill together with
# the shared MLA scaffolding so that DeepSeek-style models can
# execute on CPU.
if selected_backend and selected_backend != AttentionBackendEnum.CPU_MLA:
if selected_backend and selected_backend not in (
AttentionBackendEnum.CPU_MLA,
AttentionBackendEnum.AMX_MLA,
):
logger.info("Cannot use %s backend on CPU.", selected_backend)
logger.info_once("Using %s backend.", AttentionBackendEnum.CPU_MLA.name)
return AttentionBackendEnum.CPU_MLA.get_path()
if selected_backend and selected_backend != AttentionBackendEnum.CPU_ATTN:
logger.info("Cannot use %s backend on CPU.", selected_backend)
@@ -134,11 +151,20 @@ class CpuPlatform(Platform):
# The CPU MLA decode kernel only compiles with block_size=16 today
# (see csrc/cpu/mla_decode.cpp). If the model uses MLA we override
# the default block size regardless of user preference to avoid a
# runtime kernel dispatch failure.
# runtime kernel dispatch failure. AMX MLA has no such constraint
# (same AMX-available condition as get_attn_backend_cls), so it's
# excluded from this override.
cpu_mla_enabled = model_config is not None and getattr(
model_config, "use_mla", False
)
if cpu_mla_enabled:
amx_mla_enabled = (
cpu_mla_enabled
and cls.get_cpu_architecture() == CpuArchEnum.X86
and torch.cpu._is_amx_tile_supported()
and vllm_config.attention_config.backend != AttentionBackendEnum.CPU_MLA
)
reference_cpu_mla_enabled = cpu_mla_enabled and not amx_mla_enabled
if reference_cpu_mla_enabled:
if cache_config.user_specified_block_size and cache_config.block_size != 16:
logger.warning(
"CPU MLA backend requires block_size=16, overriding "
@@ -149,7 +175,7 @@ class CpuPlatform(Platform):
elif not cache_config.user_specified_block_size:
cache_config.block_size = 128
if not cpu_mla_enabled and cache_config.block_size % 32 != 0:
if not reference_cpu_mla_enabled and cache_config.block_size % 32 != 0:
logger.warning(
"CPU backend prefers block_size is multiples of 32, "
"otherwise the performance is not optimized."
@@ -342,7 +368,7 @@ class CpuPlatform(Platform):
vllm_config.parallel_config.tensor_parallel_size
)
if model_config is not None and model_config.use_mla:
if model_config is not None and model_config.use_mla and not amx_mla_enabled:
logger.info_once(
"MLA is enabled on a non-GPU platform; forcing chunked "
"prefill and prefix caching to be disabled."
+405
View File
@@ -0,0 +1,405 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AMX-only, high-performance MLA backend for DeepSeek V2/V3/R1 on CPU.
Built on the AMX decode/extend/bmm kernels vendored under
``csrc/cpu/sgl-kernels/``, plugged into vLLM's ``MLACommonBackend``/
``MLACommonImpl`` abstraction the same way every other concrete MLA backend
(``TritonMLAImpl``, etc.) does.
This is a separate backend from the reference ``CPUMLABackend``
(``vllm/v1/attention/backends/mla/cpu_mla.py``): that one targets every CPU
(any dtype, block_size=16, ``mla_decode_kvcache`` decode kernel + inherited
SDPA-style prefill) as a functional/CI reference, not performance. This
backend instead requires AMX (bf16 only, block_size a multiple of 32) and is
selected by the platform layer in preference to the reference backend
whenever the host supports it -- see ``CpuPlatform.get_attn_backend_cls``.
Two points where this backend differs structurally from the GPU backends,
both explained in the CPU MLA design plan:
- ``forward_mha`` is fully overridden (not inherited from
``MLACommonBaseImpl``): the GPU "compute-friendly" prefill path depends on
a pluggable ``MLAPrefillBackend`` and CUDA-only chunked-context gather ops,
neither of which exist on CPU. Instead, this attends directly in
latent-MQA space via the ``extend_attention_cpu`` kernel, which handles
cached-prefix continuation and fresh prefill in one causal pass (the KV
cache already contains the new tokens by the time this runs, since
``do_kv_cache_update`` executes first).
- Weight absorption for the prefill path is done with this impl's own
VNNI-packed copies of ``W_UK``/``W_UV`` (computed in
``process_weights_after_loading``, using ``bmm_cpu``), rather than reading
the generic ``layer.W_UK_T``/``layer.W_UV``/``layer._v_up_proj`` the way
GPU backends do, since ``forward_mha``'s abstract signature has no
``layer`` parameter (only ``forward_mqa`` does) and that signature is left
untouched. The decode path needs no such packing: ``MLAAttention.forward_impl``
already absorbs/de-absorbs Q around the ``forward_mqa`` call using the
generic (unpacked) ``layer.W_UK_T``/``layer._v_up_proj``, so
``forward_mqa`` here only has to invoke the decode kernel.
"""
from __future__ import annotations
from typing import ClassVar
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBackend,
MLACommonImpl,
MLACommonMetadata,
MLACommonMetadataBuilder,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_and_maybe_dequant_weights,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionLayer, AttentionType, MultipleOf
_MIN_WORK_PER_SPLIT = 512
_SPLIT_OCCUPANCY_MULTIPLIER = 2
def _compute_num_kv_splits(max_seq_len: int, num_threads: int) -> int:
"""Mirrors TritonMLAImpl's _compute_num_kv_splits, using the CPU thread
count in place of SM count."""
ideal_splits = 1
while ideal_splits < max(1, max_seq_len // _MIN_WORK_PER_SPLIT):
ideal_splits *= 2
max_splits = num_threads * _SPLIT_OCCUPANCY_MULTIPLIER
return min(ideal_splits, max_splits)
def _expand_block_table(block_table: torch.Tensor, block_size: int) -> torch.Tensor:
"""Adapter: vLLM's block-table paging -> the flat per-(request, position)
physical row index the decode/extend kernels expect. Called once per step
from ``AMXMLAMetadataBuilder.build``, not per layer.
"""
offsets = torch.arange(
block_size, device=block_table.device, dtype=block_table.dtype
)
flat = block_table.unsqueeze(-1) * block_size + offsets
return flat.reshape(block_table.size(0), -1).contiguous()
class AMXMLABackend(MLACommonBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
# kv_lora_rank(512) + qk_rope_head_dim(64), DeepSeek V2/V3/R1/V3.2.
return [576]
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [MultipleOf(32)]
@classmethod
def supports_block_size(cls, block_size: int | None) -> bool:
if block_size is None:
return True
return block_size % 32 == 0
@staticmethod
def get_name() -> str:
return "AMX_MLA"
@staticmethod
def get_impl_cls() -> type[AMXMLAImpl]:
return AMXMLAImpl
@staticmethod
def get_builder_cls() -> type[AMXMLAMetadataBuilder]:
return AMXMLAMetadataBuilder
class AMXMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
def build(self, common_prefix_len, common_attn_metadata, fast_build: bool = False):
attn_metadata = super().build(
common_prefix_len, common_attn_metadata, fast_build
)
block_size = self.kv_cache_spec.block_size
if attn_metadata.decode is not None:
# Built once per step and reused by every layer, instead of
# recomputing the same tensor arithmetic in every forward_mqa call.
decode = attn_metadata.decode
decode.req_to_token = _expand_block_table( # type: ignore[attr-defined]
decode.block_table, block_size
)
decode.seq_lens_i64 = decode.seq_lens.to(torch.int64) # type: ignore[attr-defined]
decode.req_pool_indices = torch.arange( # type: ignore[attr-defined]
decode.block_table.size(0),
dtype=torch.int64,
device=decode.block_table.device,
)
decode.num_kv_splits = _compute_num_kv_splits( # type: ignore[attr-defined]
attn_metadata.max_seq_len, current_platform.num_compute_units()
)
if attn_metadata.prefill is not None:
# cpu_seq_lens: total per-request context length (prefix + new
# tokens), the one thing the extend kernel needs that isn't
# already on the shared MLACommonPrefillMetadata.
prefill = attn_metadata.prefill
num_decodes = attn_metadata.num_decodes
num_prefills = attn_metadata.num_prefills
prefill.cpu_seq_lens = common_attn_metadata.seq_lens[ # type: ignore[attr-defined]
num_decodes : num_decodes + num_prefills
].to(torch.int64)
prefill.req_to_token = _expand_block_table( # type: ignore[attr-defined]
prefill.block_table, block_size
).to(torch.int64)
prefill.req_pool_indices = torch.arange( # type: ignore[attr-defined]
prefill.block_table.size(0),
dtype=torch.int64,
device=prefill.block_table.device,
)
query_start_loc_i64 = prefill.query_start_loc.to(torch.int64)
extend_seq_lens = query_start_loc_i64[1:] - query_start_loc_i64[:-1]
prefill.extend_seq_lens = extend_seq_lens # type: ignore[attr-defined]
prefill.extend_start_loc = query_start_loc_i64[:-1] # type: ignore[attr-defined]
prefill.max_len_extend = int(extend_seq_lens.max().item()) # type: ignore[attr-defined]
return attn_metadata
class AMXMLAImpl(MLACommonImpl[MLACommonMetadata]):
# Tells MLAAttention (mla_attention.py) to use this impl's own packed
# W_UK/W_UV + bmm_cpu for the shared decode absorb/de-absorb, mirroring
# is_aiter_triton_fp8_bmm_enabled/is_aiter_triton_fp4_bmm_enabled.
uses_amx_bmm = True
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: str | None,
**mla_args,
) -> None:
super().__init__(
num_heads,
head_size,
scale,
num_kv_heads,
alibi_slopes,
sliding_window,
kv_cache_dtype,
logits_soft_cap,
attn_type,
kv_sharing_target_layer_name,
**mla_args,
)
unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap]
if any(unsupported_features):
raise NotImplementedError(
"AMXMLAImpl does not support one of the following: "
"alibi_slopes, sliding_window, logits_soft_cap"
)
if attn_type != AttentionType.DECODER:
raise NotImplementedError(
"Encoder self-attention and encoder/decoder cross-attention "
"are not implemented for AMXMLAImpl"
)
self._w_uk_packed: torch.Tensor | None = None
self._w_uv_packed: torch.Tensor | None = None
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
# forward_mha (unlike forward_mqa) receives no `layer` argument, so it
# cannot read the generic layer.W_UK_T/W_UV/._v_up_proj that
# MLAAttention.process_weights_after_loading already computes. This
# impl derives its own copies from self.kv_b_proj (already available
# via the constructor) and VNNI-packs them once, ahead of time, for
# use with bmm_cpu in forward_mha.
kv_b_proj_weight = get_and_maybe_dequant_weights(
self.kv_b_proj, out_dtype=act_dtype
).T
kv_b_proj_weight = kv_b_proj_weight.view(
self.kv_lora_rank, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
)
w_uk, w_uv = kv_b_proj_weight.split(
[self.qk_nope_head_dim, self.v_head_dim], dim=-1
)
# bmm_cpu computes out[b] = mat1[b] @ mat2[b]^T (Linear-weight
# convention: mat2[b] is (OUT, IN)).
# absorb: ql_nope(N,B,L) = q_nope(N,B,P) @ w_uk_for_bmm(N,L,P)^T
# de-absorb: out(N,B,V) = attn_out(N,B,L) @ w_uv_for_bmm(N,V,L)^T
w_uk_for_bmm = w_uk.permute(1, 0, 2).contiguous() # (N, L, P)
w_uv_for_bmm = w_uv.permute(1, 2, 0).contiguous() # (N, V, L)
self._w_uk_packed = torch.ops._C.convert_weight_packed(w_uk_for_bmm)
self._w_uv_packed = torch.ops._C.convert_weight_packed(w_uv_for_bmm)
def do_kv_cache_update(
self,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
kv_cache_dtype: str,
k_scale: torch.Tensor,
) -> None:
# Overrides the default (CUDA-only concat_and_cache_mla).
if kv_cache.numel() == 0:
return
assert kv_cache_dtype == "auto", (
"AMXMLAImpl only supports an unquantized (bf16) KV cache; fp8 "
"KV cache for CPU MLA is not yet implemented."
)
ops.amx_mla_concat_and_cache(
kv_c_normed, k_pe.squeeze(1), kv_cache, slot_mapping.flatten()
)
def forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: MLACommonMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# q arrives already absorbed (tensor or (ql_nope, q_pe) tuple) --
# MLAAttention.forward_impl does the Q-absorption bmm generically
# before calling forward_mqa, using layer.W_UK_T. De-absorption is
# likewise done generically afterwards via layer._v_up_proj. Nothing
# left for this impl to do beyond invoking the decode kernel.
if isinstance(q, tuple):
q = torch.cat(q, dim=-1)
assert attn_metadata.decode is not None
num_tokens = q.shape[0]
kv_cache_flat = kv_c_and_k_pe_cache.view(-1, 1, self.head_size)
v_buffer = kv_cache_flat[..., : self.kv_lora_rank]
seq_lens = attn_metadata.decode.seq_lens_i64 # type: ignore[attr-defined]
req_to_token = attn_metadata.decode.req_to_token # type: ignore[attr-defined]
req_pool_indices = attn_metadata.decode.req_pool_indices # type: ignore[attr-defined]
num_kv_splits = attn_metadata.decode.num_kv_splits # type: ignore[attr-defined]
o = torch.zeros(
num_tokens,
self.num_heads,
self.kv_lora_rank,
dtype=q.dtype,
device=q.device,
)
attn_logits = torch.zeros(
num_tokens,
self.num_heads,
num_kv_splits,
self.kv_lora_rank + 1,
dtype=torch.float32,
device=q.device,
)
ops.cpu_mla_decode(
q,
kv_cache_flat,
v_buffer,
o,
None,
None,
None, # loc: only meaningful when key/value are given; we
# already wrote the new K/V via do_kv_cache_update before this.
attn_logits,
req_to_token,
req_pool_indices,
seq_lens,
self.scale,
0.0,
False,
0,
None,
None,
)
return o, None
def forward_mha(
self,
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: MLACommonMetadata,
k_scale: torch.Tensor,
output: torch.Tensor,
output_scale: torch.Tensor | None = None,
) -> None:
assert output_scale is None, (
"AMXMLAImpl.forward_mha does not support fused output quantization"
)
prefill = attn_metadata.prefill
assert prefill is not None
assert self._w_uk_packed is not None and self._w_uv_packed is not None
num_tokens = q.shape[0]
num_heads = self.num_heads
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
# Absorb: (N, B, P) x (N, L, P)^T -> (N, B, L)
# bmm_cpu tolerates non-contiguous mat1/out as long as their last dim
# is contiguous, so q_nope's transpose and ql_nope's transposed view
# can be passed directly -- no copy needed for either.
q_nope_t = q_nope.transpose(0, 1)
ql_nope = torch.empty(
num_tokens, num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device
)
ops.bmm_cpu(ql_nope.transpose(0, 1), q_nope_t, self._w_uk_packed, True, None)
mqa_q = torch.cat([ql_nope, q_pe], dim=-1)
kv_cache_flat = kv_c_and_k_pe_cache.view(-1, 1, self.head_size)
v_buffer = kv_cache_flat[..., : self.kv_lora_rank]
req_to_token = prefill.req_to_token # type: ignore[attr-defined]
req_pool_indices = prefill.req_pool_indices # type: ignore[attr-defined]
seq_lens = prefill.cpu_seq_lens # type: ignore[attr-defined]
extend_seq_lens = prefill.extend_seq_lens # type: ignore[attr-defined]
extend_start_loc = prefill.extend_start_loc # type: ignore[attr-defined]
max_len_extend = prefill.max_len_extend # type: ignore[attr-defined]
# k_extend/v_extend: the new tokens' own latent K/V, aliased in one
# 576-wide buffer (v_extend is a view of k_extend's first
# kv_lora_rank columns) so the kernel can gather both from one read,
# mirroring the cache's own layout.
k_extend = torch.cat(
[kv_c_normed, k_pe.reshape(num_tokens, self.qk_rope_head_dim)], dim=-1
).unsqueeze(1)
v_extend = k_extend[..., : self.kv_lora_rank]
attn_out = torch.empty(
num_tokens, num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device
)
ops.cpu_mla_extend(
mqa_q,
k_extend,
v_extend,
attn_out,
kv_cache_flat,
v_buffer,
req_to_token,
req_pool_indices,
seq_lens,
extend_seq_lens,
extend_start_loc,
max_len_extend,
self.scale,
0.0,
False,
0,
None,
None,
None,
)
# De-absorb directly into `output` (view, no extra copy on either side):
# (N, B, L) x (N, V, L)^T -> (N, B, V)
attn_out_t = attn_out.transpose(0, 1)
output_view = output.view(num_tokens, num_heads, self.v_head_dim).transpose(
0, 1
)
ops.bmm_cpu(output_view, attn_out_t, self._w_uv_packed, True, None)
@@ -0,0 +1,61 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inert MLA prefill backend for the CPU platform.
CPU's MLA attention impl (``CPUMLAImpl``) fully overrides ``forward_mha`` with a
kernel that attends directly against the paged latent KV cache (covering both
fresh prefill and cached-prefix continuation in one pass), so it never calls
into a pluggable ``MLAPrefillBackend``'s ``run_prefill_new_tokens``/
``run_prefill_context_chunk``. This class exists only to satisfy
``MLACommonMetadataBuilder``'s structural requirement that
``MLAAttention.prefill_backend`` be a real, cloneable object.
"""
from typing import TYPE_CHECKING
import torch
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
if TYPE_CHECKING:
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonPrefillMetadata,
)
class CPUNativeMLAPrefillBackend(MLAPrefillBackend):
"""Placeholder MLA prefill backend for CPU; never actually invoked."""
@staticmethod
def get_name() -> str:
return "CPU_NATIVE"
def run_prefill_new_tokens(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
return_softmax_lse: bool,
out: torch.Tensor | None = None,
output_scale: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
raise AssertionError(
"CPUNativeMLAPrefillBackend.run_prefill_new_tokens is unreachable: "
"CPUMLAImpl.forward_mha fully overrides the dense-MHA prefill path "
"and never calls the pluggable prefill backend."
)
def run_prefill_context_chunk(
self,
chunk: "MLACommonPrefillMetadata.ContextChunk",
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
raise AssertionError(
"CPUNativeMLAPrefillBackend.run_prefill_context_chunk is "
"unreachable: CPUMLAImpl.forward_mha fully overrides the "
"dense-MHA prefill path and never calls the pluggable prefill "
"backend."
)
@@ -52,6 +52,9 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
"vllm.v1.attention.backends.mla.prefill.aiter_flash_attn."
"AiterFlashAttnPrefillBackend"
)
CPU_NATIVE = (
"vllm.v1.attention.backends.mla.prefill.cpu_native.CPUNativeMLAPrefillBackend"
)
# Placeholder for third-party/custom backends - must be registered before use
# set to None to avoid alias with other backend, whose value is an empty string
CUSTOM = None
@@ -109,6 +109,8 @@ def get_mla_prefill_backend(
device_capability = current_platform.get_device_capability()
if device_capability is None:
if current_platform.is_cpu():
return MLAPrefillBackendEnum.CPU_NATIVE.get_class()
logger.info_once(
"Device capability not available, using FlashAttention MLA prefill backend."
)
+1
View File
@@ -123,6 +123,7 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
)
CPU_ATTN = "vllm.v1.attention.backends.cpu_attn.CPUAttentionBackend"
CPU_MLA = "vllm.v1.attention.backends.mla.cpu_mla.CPUMLABackend"
AMX_MLA = "vllm.v1.attention.backends.mla.amx_mla.AMXMLABackend"
TURBOQUANT = "vllm.v1.attention.backends.turboquant_attn.TurboQuantAttentionBackend"
# Placeholder for third-party/custom backends - must be registered before use
# set to None to avoid alias with other backend, whose value is an empty string