[Model] Add DeepSeek-V4 CPU backend (#55355)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-09-10 14:14:48 +08:00
committed by GitHub
parent 49632fb3ac
commit c3ccc0e957
47 changed files with 9564 additions and 627 deletions
+30
View File
@@ -343,6 +343,36 @@ inline void parallel_2d(int m, int n, const func_t& f) {
#endif
}
// Like parallel_2d but with explicit nth_m x nth_n decomposition.
// Caller is responsible for choosing nth_m and nth_n.
//
// Cherry-picked from a newer sglang commit than this file's last full sync
// (needed by mhc.cpp's phase-1 tiling); not part of a full common.h resync.
template <typename func_t>
inline void parallel_2d_tiled(int m, int n, int nth_m, int nth_n, const func_t& f) {
const int nth = nth_m * nth_n;
#if defined(_OPENMP)
#pragma omp parallel num_threads(nth)
{
int ith = omp_get_thread_num();
int ith_m = ith / nth_n;
int ith_n = ith % nth_n;
int thread_block_m = div_up(m, nth_m);
int thread_block_n = div_up(n, nth_n);
int begin_m = std::min(ith_m * thread_block_m, m);
int end_m = std::min(begin_m + thread_block_m, m);
int begin_n = std::min(ith_n * thread_block_n, n);
int end_n = std::min(begin_n + thread_block_n, n);
f(begin_m, end_m, begin_n, end_n);
}
#else
f(0, m, 0, n);
#endif
}
// limit max cache blocks
// when we need to do pre-unpack for weights, e.g. fp8
#define MAX_CACHE_BLOCK_SIZE 4
+747
View File
@@ -0,0 +1,747 @@
// Adapted from
// https://github.com/sgl-project/sglang/blob/main/python/sglang/kernels/aot/csrc/cpu/compressor.cpp
//
// DeepSeek-V4 compressor: softmax-weighted pooling over a sliding window of
// raw KV/score state, RMSNorm, FP8/UE8M0 quant (NoPE) + GPT-J RoPE (RoPE),
// paged-insert into the fp8_ds_mla cache layout (see store_cache.cpp).
// Covers both the head_dim=512 attention-compressor path
// (`compress_norm_rope_store_cpu`) and the head_dim=128 indexer-compressor
// path (`compress_norm_rope_store_indexer_cpu`, distinct cache layout --
// see its own comment below).
//
// Differences from upstream:
// - Upstream splits decode (`compress_decode_cpu`) from prefill/extend
// (eager `compress_extend_separate`); this port stays unified across
// decode/prefill/extend, one program per token, matching vLLM's own
// GPU/Triton reference (`_fused_kv_compress_norm_rope_insert_sparse_attn`,
// common/ops/fused_compress_quant_cache.py).
// - Window-gather indices arrive pre-resolved to flat state-cache row ids
// from the Python caller (mirroring flash_mla.cpp), not looked up via
// block_table in-kernel.
// - The state cache is pooled with the main KV cache's physical pages
// (CompressorStateCache in compressor.py), so its per-block byte stride
// is page-aligned, not `block_size * row_width` -- row ids are addressed
// as (block_idx, pos_in_block) against the tensor's real block stride,
// never by flattening to a 2D view.
//
// clang-format off
#include <algorithm>
#include <array>
#include <cmath>
#include <limits>
#include "common.h"
#include "vec.h"
#if !defined(CPU_CAPABILITY_AVX512)
static_assert(false,
"compressor.cpp is only ever compiled into the AVX512+AMX-flagged "
"_C target (see cmake/cpu_extension.cmake's VLLM_EXT_SRC_SGL) and "
"relies on that unconditionally -- CPU_CAPABILITY_AVX512 must be "
"defined here.");
#endif
namespace {
using fVec = at::vec::Vectorized<float>;
constexpr int64_t kNopeDim = 448;
constexpr int64_t kRopeDim = 64;
constexpr int64_t kHeadDim = kNopeDim + kRopeDim;
constexpr int64_t kQuantBlock = 64;
constexpr int64_t kNumQuantBlocks = kNopeDim / kQuantBlock;
constexpr int64_t kScaleBytesPerToken = kNumQuantBlocks + 1;
constexpr int64_t kTokenDataBytes = kNopeDim + kRopeDim * 2;
constexpr float kFp8Max = 448.0f;
constexpr float kFp8Min = -448.0f;
constexpr float kQuantEps = 1e-4f;
// In-place interleaved (GPT-J-style) RoPE: x[2i]/x[2i+1] are one (even, odd)
// pair, cos[i]/sin[i] its angle. Adapted from SGLang's `rope.cpp`
// (`apply_rotary_emb_row`), which avoids gather/scatter (slow on AVX512) by
// broadcasting cos/sin across each pair via in-register permutes instead:
// `_mm512_permute_ps(xv, 0xB1)` swaps the (even, odd) lanes within every
// 2-float pair in one shuffle, and cos/sin -- loaded as 8 contiguous values
// -- are duplicated pairwise into the same layout via `_mm512_permutexvar_ps`.
// SGLang's version reads cos/sin pre-interleaved with x (`[c0,s0,c1,s1,...]`)
// since its freqs tensor is laid out that way; ours are two separate flat
// arrays (`cos[i]`, `sin[i]`, matching store_cache.cpp's convention), hence
// the extra broadcast-duplicate permute here instead of a plain load.
// Both call sites in this file pass half_dim=32 (kRopeDim/2, kIndexerRopeDim/2
// are both 64/2), an exact multiple of 8, so this file's build (always AVX512,
// see file header) never needs a scalar remainder.
inline void apply_gptj_rope_inplace(float* __restrict__ x, const float* __restrict__ cos, const float* __restrict__ sin, int64_t half_dim) {
// Duplicates each of 8 source lanes into 2 adjacent output lanes:
// [c0,c1,...,c7] -> [c0,c0,c1,c1,...,c7,c7].
const __m512i dup_idx = _mm512_set_epi32(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0);
// Forward GPT-J rotation negates the sin broadcast on the even (real)
// lane of each pair; the odd (imag) lane keeps sin's sign.
const __m512 sign_mask = _mm512_castsi512_ps(_mm512_set_epi32(
0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000,
0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000));
for (int64_t i = 0; i + 8 <= half_dim; i += 8) {
__m512 xv = _mm512_loadu_ps(x + 2 * i);
__m512 xv_swapped = _mm512_permute_ps(xv, 0xB1);
__m512 cos_b = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(cos + i)));
__m512 sin_b = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(sin + i)));
__m512 sin_signed = _mm512_xor_ps(sin_b, sign_mask);
__m512 out = _mm512_fmadd_ps(xv, cos_b, _mm512_mul_ps(xv_swapped, sin_signed));
_mm512_storeu_ps(x + 2 * i, out);
}
}
// AVX512 tile-quantization, adapted verbatim from SGLang's `store_cache.cpp`
// (`quant_to_nope_fp8_rope_bf16_pack_cpu`'s
// `cvt_fp32x16_to_fp8x16`/`quantize_tile_avx512`) -- SGLang itself keeps
// `compress_decode_cpu` fp32-output-only and calls this quant step as a
// separate op; vLLM's compressor kernel stays fused (per this file's header
// comment), so the same AVX512 tile-quant math is inlined here instead.
#if defined(CPU_CAPABILITY_AVX512)
// Vectorized float32x16 -> fp8_e4m3fn x16 using AVX512 bit manipulation.
// Implements RNE (round-to-nearest-even). Flushes subnormal fp8 to zero.
// fp8_e4m3fn: sign(1), exp(4, bias=7), mant(3), no inf/nan special values.
// Normal: exp_field in [1..15], value = 2^(exp-7) * (1 + mant/8)
// Input assumed clamped to [-448, 448].
inline __m128i cvt_fp32x16_to_fp8x16(__m512 input) {
__m512i bits = _mm512_castps_si512(input);
__m512i signs = _mm512_srli_epi32(_mm512_and_si512(bits, _mm512_set1_epi32(0x80000000)), 24);
__m512i abs_bits = _mm512_and_si512(bits, _mm512_set1_epi32(0x7FFFFFFF));
__m512i f32_exp = _mm512_srli_epi32(abs_bits, 23);
__m512i f32_mant = _mm512_and_si512(abs_bits, _mm512_set1_epi32(0x7FFFFF));
__m512i fp8_exp = _mm512_sub_epi32(f32_exp, _mm512_set1_epi32(120));
__m512i mant3 = _mm512_srli_epi32(f32_mant, 20);
__m512i round_bit = _mm512_and_si512(_mm512_srli_epi32(f32_mant, 19), _mm512_set1_epi32(1));
__m512i sticky_bits = _mm512_and_si512(f32_mant, _mm512_set1_epi32(0x7FFFF));
__mmask16 has_sticky = _mm512_cmpneq_epi32_mask(sticky_bits, _mm512_setzero_si512());
__m512i lsb = _mm512_and_si512(mant3, _mm512_set1_epi32(1));
__m512i sticky_or_lsb = _mm512_or_si512(_mm512_maskz_mov_epi32(has_sticky, _mm512_set1_epi32(1)), lsb);
__m512i do_round = _mm512_and_si512(round_bit, sticky_or_lsb);
mant3 = _mm512_add_epi32(mant3, do_round);
__mmask16 mant_ovf = _mm512_cmpeq_epi32_mask(mant3, _mm512_set1_epi32(8));
mant3 = _mm512_mask_mov_epi32(mant3, mant_ovf, _mm512_setzero_si512());
fp8_exp = _mm512_mask_add_epi32(fp8_exp, mant_ovf, fp8_exp, _mm512_set1_epi32(1));
fp8_exp = _mm512_min_epi32(fp8_exp, _mm512_set1_epi32(15));
__m512i result = _mm512_or_si512(_mm512_slli_epi32(fp8_exp, 3), mant3);
__mmask16 is_subnormal_or_zero = _mm512_cmplt_epi32_mask(f32_exp, _mm512_set1_epi32(121));
result = _mm512_mask_mov_epi32(result, is_subnormal_or_zero, _mm512_setzero_si512());
result = _mm512_or_si512(result, signs);
return _mm512_cvtepi32_epi8(result);
}
// Process one 64-wide tile: find amax, compute a shared power-of-two scale,
// quantize to fp8. Returns the UE8M0 scale byte for the tile.
inline uint8_t quantize_tile_avx512(const at::BFloat16* __restrict__ src, uint8_t* __restrict__ dst) {
__m256i bf16_0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src));
__m256i bf16_1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 16));
__m256i bf16_2 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 32));
__m256i bf16_3 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 48));
__m512 f0 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_0), 16));
__m512 f1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_1), 16));
__m512 f2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_2), 16));
__m512 f3 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_3), 16));
const __m512i abs_mask_i = _mm512_set1_epi32(0x7FFFFFFF);
__m512 abs0 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f0), abs_mask_i));
__m512 abs1 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f1), abs_mask_i));
__m512 abs2 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f2), abs_mask_i));
__m512 abs3 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f3), abs_mask_i));
__m512 max01 = _mm512_max_ps(abs0, abs1);
__m512 max23 = _mm512_max_ps(abs2, abs3);
__m512 max0123 = _mm512_max_ps(max01, max23);
float amax = _mm512_reduce_max_ps(max0123);
float scale = std::max(amax / kFp8Max, kQuantEps);
float ceil_log2 = std::ceil(std::log2(scale));
float scale_pow2 = std::exp2(ceil_log2);
float scale_inv = 1.0f / scale_pow2;
int exponent = static_cast<int>(ceil_log2);
uint8_t scale_uint8 = static_cast<uint8_t>(exponent + 127);
__m512 vinv = _mm512_set1_ps(scale_inv);
__m512 s0 = _mm512_mul_ps(f0, vinv);
__m512 s1 = _mm512_mul_ps(f1, vinv);
__m512 s2 = _mm512_mul_ps(f2, vinv);
__m512 s3 = _mm512_mul_ps(f3, vinv);
__m512 vmax = _mm512_set1_ps(kFp8Max);
__m512 vmin = _mm512_set1_ps(kFp8Min);
s0 = _mm512_max_ps(_mm512_min_ps(s0, vmax), vmin);
s1 = _mm512_max_ps(_mm512_min_ps(s1, vmax), vmin);
s2 = _mm512_max_ps(_mm512_min_ps(s2, vmax), vmin);
s3 = _mm512_max_ps(_mm512_min_ps(s3, vmax), vmin);
__m128i fp8_0 = cvt_fp32x16_to_fp8x16(s0);
__m128i fp8_1 = cvt_fp32x16_to_fp8x16(s1);
__m128i fp8_2 = cvt_fp32x16_to_fp8x16(s2);
__m128i fp8_3 = cvt_fp32x16_to_fp8x16(s3);
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), fp8_0);
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), fp8_1);
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), fp8_2);
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), fp8_3);
// Patch subnormal-magnitude lanes: `cvt_fp32x16_to_fp8x16` flushes
// anything below the smallest fp8_e4m3fn normal (2^-6) to zero, but
// vLLM's dequant path relies on `at::Float8_e4m3fn`'s real subnormal
// encoding -- redo just those rare lanes with the exact scalar
// conversion to stay consistent with the read side.
alignas(64) float scaled[kQuantBlock];
_mm512_store_ps(scaled, s0);
_mm512_store_ps(scaled + 16, s1);
_mm512_store_ps(scaled + 32, s2);
_mm512_store_ps(scaled + 48, s3);
for (int64_t j = 0; j < kQuantBlock; ++j) {
const float av = std::abs(scaled[j]);
if (av != 0.f && av < 0x1p-6f) {
dst[j] = at::Float8_e4m3fn(scaled[j]).x;
}
}
return scale_uint8;
}
#endif // CPU_CAPABILITY_AVX512
void save_partial_states_impl(
float* __restrict__ state_cache, // [num_blocks, block_stride]
int64_t block_stride, // element stride between blocks
int64_t block_size, // rows (slots) per block
const float* __restrict__ kv, // [num_tokens, state_width]
int64_t kv_row_stride, // element stride between kv rows
const float* __restrict__ score, // [num_tokens, state_width]
int64_t score_row_stride, // element stride between score rows
const float* __restrict__ ape, // [compress_ratio, state_width]
const int64_t* __restrict__ positions, // [num_tokens]
const int64_t* __restrict__ slot_mapping, // [num_tokens]
int64_t state_width,
int64_t compress_ratio,
int64_t num_tokens) {
const int64_t row_width = 2 * state_width;
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
for (int64_t t = begin; t < end; ++t) {
const int64_t slot_id = slot_mapping[t];
if (slot_id < 0) continue;
const int64_t block_idx = slot_id / block_size;
const int64_t pos_in_block = slot_id % block_size;
float* const row = state_cache + block_idx * block_stride + pos_in_block * row_width;
const float* const kv_t = kv + t * kv_row_stride;
// state_width is always a multiple of 16 for this model (512/1024 for
// the main compressor, 128/256 for the indexer compressor).
for (int64_t i = 0; i + 16 <= state_width; i += 16) {
fVec::loadu(kv_t + i).store(row + i);
}
const int64_t position = positions[t];
const float* const ape_row = ape + (position % compress_ratio) * state_width;
const float* const score_t = score + t * score_row_stride;
float* const score_out = row + state_width;
for (int64_t i = 0; i + 16 <= state_width; i += 16) {
(fVec::loadu(score_t + i) + fVec::loadu(ape_row + i)).store(score_out + i);
}
}
});
}
void compress_norm_rope_store_impl(
const float* __restrict__ state_cache, // [num_blocks, state_block_stride]
int64_t state_block_stride, // element stride between state blocks
int64_t state_block_size, // rows (slots) per state block
const int64_t* __restrict__ gather_slots, // [num_tokens, window], -1 = invalid
const int64_t* __restrict__ positions, // [num_tokens]
const int64_t* __restrict__ kv_slot_mapping, // [num_tokens], -1 = skip
const float* __restrict__ rms_norm_weight, // [kHeadDim]
float rms_norm_eps,
const float* __restrict__ cos_sin_cache, // [max_pos, kRopeDim]
uint8_t* __restrict__ kv_cache, // [num_blocks, cache_row_stride]
int64_t cache_row_stride,
int64_t kv_cache_block_size,
int64_t compress_ratio,
int64_t window,
int64_t state_width,
int64_t num_tokens) {
const int64_t rope_half = kRopeDim / 2;
const int64_t state_row_width = 2 * state_width;
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
std::array<float, kHeadDim> max_v;
std::array<float, kHeadDim> denom;
std::array<float, kHeadDim> acc;
std::array<float, kHeadDim> normed;
std::array<at::BFloat16, kNopeDim> nope_bf16;
for (int64_t t = begin; t < end; ++t) {
const int64_t position = positions[t];
if ((position + 1) % compress_ratio != 0) continue;
const int64_t kv_slot = kv_slot_mapping[t];
if (kv_slot < 0) continue;
std::fill(max_v.begin(), max_v.end(), -std::numeric_limits<float>::infinity());
std::fill(denom.begin(), denom.end(), 0.f);
std::fill(acc.begin(), acc.end(), 0.f);
// Per-channel (not per-position) softmax over the window: each of the
// kHeadDim columns is normalized independently across the window axis.
for (int64_t w = 0; w < window; ++w) {
const int64_t slot = gather_slots[t * window + w];
if (slot < 0) continue;
const int64_t block_idx = slot / state_block_size;
const int64_t pos_in_block = slot % state_block_size;
const int64_t head_offset = (w >= compress_ratio) ? kHeadDim : 0;
const float* const score_row = state_cache + block_idx * state_block_stride +
pos_in_block * state_row_width + head_offset + state_width;
for (int64_t d = 0; d < kHeadDim; d += 16) {
__m512 mx = _mm512_loadu_ps(max_v.data() + d);
__m512 sv = _mm512_loadu_ps(score_row + d);
_mm512_storeu_ps(max_v.data() + d, _mm512_max_ps(mx, sv));
}
}
for (int64_t w = 0; w < window; ++w) {
const int64_t slot = gather_slots[t * window + w];
if (slot < 0) continue;
const int64_t block_idx = slot / state_block_size;
const int64_t pos_in_block = slot % state_block_size;
const int64_t head_offset = (w >= compress_ratio) ? kHeadDim : 0;
const float* const row = state_cache + block_idx * state_block_stride +
pos_in_block * state_row_width + head_offset;
const float* const score_row = row + state_width;
for (int64_t d = 0; d < kHeadDim; d += 16) {
fVec e_v = (fVec::loadu(score_row + d) - fVec::loadu(max_v.data() + d)).exp();
(fVec::loadu(denom.data() + d) + e_v).store(denom.data() + d);
(fVec::loadu(acc.data() + d) + e_v * fVec::loadu(row + d)).store(acc.data() + d);
}
}
float sq_sum = 0.f;
{
fVec sq_sum_v(0.f);
for (int64_t d = 0; d < kHeadDim; d += 16) {
fVec v = fVec::loadu(acc.data() + d) / fVec::loadu(denom.data() + d);
v.store(normed.data() + d);
sq_sum_v = sq_sum_v + v * v;
}
sq_sum = vec_reduce_sum(sq_sum_v);
}
const float inv_rms = 1.0f / std::sqrt(sq_sum / static_cast<float>(kHeadDim) + rms_norm_eps);
{
fVec inv_rms_v(inv_rms);
for (int64_t d = 0; d < kHeadDim; d += 16) {
(fVec::loadu(normed.data() + d) * inv_rms_v * fVec::loadu(rms_norm_weight + d)).store(normed.data() + d);
}
}
// bf16 round-trip feeds only the NoPE FP8 quantization (RoPE rotates
// the un-rounded `normed` below, then casts once to bf16 at store
// time for the separate bf16 rope region).
for (int64_t d = 0; d < kNopeDim; d += 16)
store_from_float_ext<at::BFloat16>(nope_bf16.data() + d, fVec::loadu(normed.data() + d));
const int64_t compressed_pos = (position / compress_ratio) * compress_ratio;
const float* const cos = cos_sin_cache + compressed_pos * kRopeDim;
const float* const sin = cos + rope_half;
apply_gptj_rope_inplace(normed.data() + kNopeDim, cos, sin, rope_half);
const int64_t block_idx = kv_slot / kv_cache_block_size;
const int64_t pos_in_block = kv_slot % kv_cache_block_size;
uint8_t* const block_base = kv_cache + block_idx * cache_row_stride;
uint8_t* const token_fp8_ptr = block_base + pos_in_block * kTokenDataBytes;
uint8_t* const token_bf16_ptr = token_fp8_ptr + kNopeDim;
uint8_t* const token_scale_ptr =
block_base + kv_cache_block_size * kTokenDataBytes + pos_in_block * kScaleBytesPerToken;
// AVX512 tile-quantization (SGLang's `quantize_tile_avx512`), one
// 64-wide tile per iteration.
for (int64_t qb = 0; qb < kNumQuantBlocks; ++qb) {
const at::BFloat16* tile_src = nope_bf16.data() + qb * kQuantBlock;
uint8_t* tile_dst = token_fp8_ptr + qb * kQuantBlock;
token_scale_ptr[qb] = quantize_tile_avx512(tile_src, tile_dst);
}
token_scale_ptr[kNumQuantBlocks] = 0;
auto* const out_rope = reinterpret_cast<at::BFloat16*>(token_bf16_ptr);
for (int64_t ri = 0; ri < kRopeDim; ri += 16)
store_from_float_ext<at::BFloat16>(out_rope + ri, fVec::loadu(normed.data() + kNopeDim + ri));
}
});
}
// Indexer compressor (head_dim=128): same softmax-pool + RMSNorm + GPT-J
// RoPE prologue as the head=512 path above, but a different cache layout:
// all 128 dims are FP8 in one combined quant block (no separate bf16 rope
// region, single raw fp32 scale per token) instead of 7 UE8M0 blocks over
// nope alone -- matching the paged indexer K-cache layout
// fp8_paged_mqa_logits_cpu/topk_transform_512_cpu (paged_mqa_logits.cpp/
// topk.cpp) read, and the GPU/Triton reference
// (`_fused_kv_compress_norm_rope_insert_indexer_attn`,
// common/ops/fused_compress_quant_cache.py) exactly.
constexpr int64_t kIndexerHeadDim = 128;
constexpr int64_t kIndexerRopeDim = 64;
constexpr int64_t kIndexerNopeDim = kIndexerHeadDim - kIndexerRopeDim;
constexpr int64_t kIndexerTokenBytes = kIndexerHeadDim;
constexpr int64_t kIndexerScaleBytes = 4;
void compress_norm_rope_store_indexer_impl(
const float* __restrict__ state_cache, // [num_blocks, state_block_stride]
int64_t state_block_stride,
int64_t state_block_size,
const int64_t* __restrict__ gather_slots, // [num_tokens, window], -1 = invalid
const int64_t* __restrict__ positions, // [num_tokens]
const int64_t* __restrict__ kv_slot_mapping, // [num_tokens], -1 = skip
const float* __restrict__ rms_norm_weight, // [kIndexerHeadDim]
float rms_norm_eps,
const float* __restrict__ cos_sin_cache, // [max_pos, kIndexerRopeDim]
uint8_t* __restrict__ kv_cache, // [num_blocks, cache_row_stride]
int64_t cache_row_stride,
int64_t kv_cache_block_size,
int64_t compress_ratio,
int64_t window,
int64_t state_width,
int64_t num_tokens) {
const int64_t rope_half = kIndexerRopeDim / 2;
const int64_t state_row_width = 2 * state_width;
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
std::array<float, kIndexerHeadDim> max_v;
std::array<float, kIndexerHeadDim> denom;
std::array<float, kIndexerHeadDim> acc;
std::array<float, kIndexerHeadDim> normed;
std::array<at::BFloat16, kIndexerHeadDim> quant_bf16;
for (int64_t t = begin; t < end; ++t) {
const int64_t position = positions[t];
if ((position + 1) % compress_ratio != 0) continue;
const int64_t kv_slot = kv_slot_mapping[t];
if (kv_slot < 0) continue;
std::fill(max_v.begin(), max_v.end(), -std::numeric_limits<float>::infinity());
std::fill(denom.begin(), denom.end(), 0.f);
std::fill(acc.begin(), acc.end(), 0.f);
for (int64_t w = 0; w < window; ++w) {
const int64_t slot = gather_slots[t * window + w];
if (slot < 0) continue;
const int64_t block_idx = slot / state_block_size;
const int64_t pos_in_block = slot % state_block_size;
const int64_t head_offset = (w >= compress_ratio) ? kIndexerHeadDim : 0;
const float* const score_row = state_cache + block_idx * state_block_stride +
pos_in_block * state_row_width + head_offset + state_width;
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
__m512 mx = _mm512_loadu_ps(max_v.data() + d);
__m512 sv = _mm512_loadu_ps(score_row + d);
_mm512_storeu_ps(max_v.data() + d, _mm512_max_ps(mx, sv));
}
}
for (int64_t w = 0; w < window; ++w) {
const int64_t slot = gather_slots[t * window + w];
if (slot < 0) continue;
const int64_t block_idx = slot / state_block_size;
const int64_t pos_in_block = slot % state_block_size;
const int64_t head_offset = (w >= compress_ratio) ? kIndexerHeadDim : 0;
const float* const row = state_cache + block_idx * state_block_stride +
pos_in_block * state_row_width + head_offset;
const float* const score_row = row + state_width;
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
fVec e_v = (fVec::loadu(score_row + d) - fVec::loadu(max_v.data() + d)).exp();
(fVec::loadu(denom.data() + d) + e_v).store(denom.data() + d);
(fVec::loadu(acc.data() + d) + e_v * fVec::loadu(row + d)).store(acc.data() + d);
}
}
float sq_sum = 0.f;
{
fVec sq_sum_v(0.f);
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
fVec v = fVec::loadu(acc.data() + d) / fVec::loadu(denom.data() + d);
v.store(normed.data() + d);
sq_sum_v = sq_sum_v + v * v;
}
sq_sum = vec_reduce_sum(sq_sum_v);
}
const float inv_rms = 1.0f / std::sqrt(sq_sum / static_cast<float>(kIndexerHeadDim) + rms_norm_eps);
{
fVec inv_rms_v(inv_rms);
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
(fVec::loadu(normed.data() + d) * inv_rms_v * fVec::loadu(rms_norm_weight + d)).store(normed.data() + d);
}
}
const int64_t compressed_pos = (position / compress_ratio) * compress_ratio;
const float* const cos = cos_sin_cache + compressed_pos * kIndexerRopeDim;
const float* const sin = cos + rope_half;
apply_gptj_rope_inplace(normed.data() + kIndexerNopeDim, cos, sin, rope_half);
// Indexer path rounds the FULL post-RoPE vector through bf16 before
// quantizing (single combined block); the head=512 path above instead
// rounds only the NoPE region before RoPE (separate bf16 rope store).
for (int64_t d = 0; d < kIndexerHeadDim; d += 16)
store_from_float_ext<at::BFloat16>(quant_bf16.data() + d, fVec::loadu(normed.data() + d));
const int64_t block_idx = kv_slot / kv_cache_block_size;
const int64_t pos_in_block = kv_slot % kv_cache_block_size;
uint8_t* const block_base = kv_cache + block_idx * cache_row_stride;
uint8_t* const token_fp8_ptr = block_base + pos_in_block * kIndexerTokenBytes;
uint8_t* const token_scale_ptr =
block_base + kv_cache_block_size * kIndexerTokenBytes + pos_in_block * kIndexerScaleBytes;
float absmax = 1e-4f;
{
__m512 max_v_reg = _mm512_set1_ps(absmax);
for (int64_t di = 0; di < kIndexerHeadDim; di += 16) {
__m256i bf16_v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(quant_bf16.data() + di));
__m512 f = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_v), 16));
__m512 av = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f), _mm512_set1_epi32(0x7FFFFFFF)));
max_v_reg = _mm512_max_ps(max_v_reg, av);
}
absmax = _mm512_reduce_max_ps(max_v_reg);
}
const float exponent = std::ceil(std::log2(absmax / kFp8Max));
const float scale = std::exp2(exponent);
const float inv_scale = 1.0f / scale;
// AVX512 quantize+store, reusing SGLang's `cvt_fp32x16_to_fp8x16` bit-
// manipulation conversion (this single-shared-scale 128-wide block has
// no direct SGLang equivalent -- see file header -- so only the fp8
// conversion primitive is ported, the amax/scale formula is unchanged
// from vLLM's own reference).
uint8_t* const out_fp8 = token_fp8_ptr;
{
const __m512 vinv = _mm512_set1_ps(inv_scale);
const __m512 vmax = _mm512_set1_ps(kFp8Max);
const __m512 vmin = _mm512_set1_ps(kFp8Min);
alignas(64) float scaled_buf[kIndexerHeadDim];
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
__m256i bf16_v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(quant_bf16.data() + d));
__m512 f = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_v), 16));
__m512 s = _mm512_mul_ps(f, vinv);
s = _mm512_max_ps(_mm512_min_ps(s, vmax), vmin);
_mm512_store_ps(scaled_buf + d, s);
__m128i fp8_v = cvt_fp32x16_to_fp8x16(s);
_mm_storeu_si128(reinterpret_cast<__m128i*>(out_fp8 + d), fp8_v);
}
// Patch subnormal-magnitude lanes (see `quantize_tile_avx512`'s
// comment above for why the vectorized conversion's flush-to-zero
// must not be trusted here).
for (int64_t d = 0; d < kIndexerHeadDim; ++d) {
const float av = std::abs(scaled_buf[d]);
if (av != 0.f && av < 0x1p-6f) {
out_fp8[d] = at::Float8_e4m3fn(scaled_buf[d]).x;
}
}
}
// Raw fp32 scale (not UE8M0-encoded) -- single quant block, matches
// fp8_paged_mqa_logits_cpu's (paged_mqa_logits.cpp) read-side expectations.
*reinterpret_cast<float*>(token_scale_ptr) = scale;
}
});
}
} // anonymous namespace
void save_partial_states_cpu(
at::Tensor& kv,
at::Tensor& score,
at::Tensor& ape,
at::Tensor& positions,
at::Tensor& state_cache,
at::Tensor& slot_mapping) {
TORCH_CHECK(
kv.dim() == 2 && kv.scalar_type() == at::kFloat, "save_partial_states_cpu: kv must be a 2D fp32 tensor");
TORCH_CHECK(score.sizes() == kv.sizes() && score.scalar_type() == at::kFloat,
"save_partial_states_cpu: score must match kv's shape/dtype");
TORCH_CHECK(ape.dim() == 2 && ape.size(1) == kv.size(1) && ape.scalar_type() == at::kFloat,
"save_partial_states_cpu: ape must be [compress_ratio, state_width] fp32");
TORCH_CHECK(positions.scalar_type() == at::kLong && positions.numel() == kv.size(0),
"save_partial_states_cpu: positions must be int64 with num_tokens entries");
TORCH_CHECK(slot_mapping.scalar_type() == at::kLong && slot_mapping.numel() == kv.size(0),
"save_partial_states_cpu: slot_mapping must be int64 with num_tokens entries");
TORCH_CHECK(state_cache.dim() == 3 && state_cache.size(2) == 2 * kv.size(1) &&
state_cache.scalar_type() == at::kFloat,
"save_partial_states_cpu: state_cache must be [num_blocks, block_size, "
"2*state_width] fp32");
TORCH_CHECK(state_cache.stride(2) == 1 && state_cache.stride(1) == state_cache.size(2),
"save_partial_states_cpu: state_cache rows must be contiguous within a block");
TORCH_CHECK(
kv.device().is_cpu() && score.device().is_cpu() && ape.device().is_cpu() && positions.device().is_cpu() &&
state_cache.device().is_cpu() && slot_mapping.device().is_cpu(),
"save_partial_states_cpu: all inputs must be CPU tensors");
const int64_t num_tokens = kv.size(0);
const int64_t state_width = kv.size(1);
const int64_t compress_ratio = ape.size(0);
const int64_t block_size = state_cache.size(1);
TORCH_CHECK(
kv.stride(1) == 1 && score.stride(1) == 1 && ape.is_contiguous() && positions.is_contiguous() &&
slot_mapping.is_contiguous(),
"save_partial_states_cpu: kv/score must be contiguous in their last dim "
"(row stride may exceed state_width -- e.g. a column-split view of a "
"fused kv_score projection); ape/positions/slot_mapping must be fully "
"contiguous");
save_partial_states_impl(
state_cache.data_ptr<float>(),
state_cache.stride(0),
block_size,
kv.data_ptr<float>(),
kv.stride(0),
score.data_ptr<float>(),
score.stride(0),
ape.data_ptr<float>(),
positions.data_ptr<int64_t>(),
slot_mapping.data_ptr<int64_t>(),
state_width,
compress_ratio,
num_tokens);
}
void compress_norm_rope_store_cpu(
at::Tensor& state_cache,
at::Tensor& gather_slots,
at::Tensor& positions,
at::Tensor& kv_slot_mapping,
at::Tensor& rms_norm_weight,
double rms_norm_eps,
at::Tensor& cos_sin_cache,
at::Tensor& kv_cache_2d,
int64_t kv_cache_block_size,
int64_t compress_ratio) {
TORCH_CHECK(state_cache.dim() == 3 && state_cache.scalar_type() == at::kFloat,
"compress_norm_rope_store_cpu: state_cache must be a 3D fp32 tensor");
TORCH_CHECK(state_cache.stride(2) == 1 && state_cache.stride(1) == state_cache.size(2),
"compress_norm_rope_store_cpu: state_cache rows must be contiguous within a block");
TORCH_CHECK(gather_slots.dim() == 2 && gather_slots.scalar_type() == at::kLong,
"compress_norm_rope_store_cpu: gather_slots must be a 2D int64 tensor");
const int64_t num_tokens = gather_slots.size(0);
TORCH_CHECK(positions.scalar_type() == at::kLong && positions.numel() == num_tokens,
"compress_norm_rope_store_cpu: positions must be int64 with num_tokens entries");
TORCH_CHECK(kv_slot_mapping.scalar_type() == at::kLong && kv_slot_mapping.numel() == num_tokens,
"compress_norm_rope_store_cpu: kv_slot_mapping must be int64 with num_tokens entries");
TORCH_CHECK(rms_norm_weight.numel() == kHeadDim,
"compress_norm_rope_store_cpu: rms_norm_weight must have head_dim (512) entries");
TORCH_CHECK(kv_cache_2d.dim() == 2 && kv_cache_2d.scalar_type() == at::kByte,
"compress_norm_rope_store_cpu: kv_cache_2d must be a 2D uint8 tensor");
TORCH_CHECK(state_cache.size(2) % 2 == 0, "compress_norm_rope_store_cpu: state_cache width must be even");
TORCH_CHECK(
rms_norm_weight.scalar_type() == at::kFloat && rms_norm_weight.is_contiguous() &&
cos_sin_cache.scalar_type() == at::kFloat && cos_sin_cache.is_contiguous(),
"compress_norm_rope_store_cpu: rms_norm_weight/cos_sin_cache must be "
"float32 and contiguous (convert once in Python)");
TORCH_CHECK(
gather_slots.is_contiguous() && positions.is_contiguous() && kv_slot_mapping.is_contiguous(),
"compress_norm_rope_store_cpu: gather_slots/positions/kv_slot_mapping must be contiguous");
TORCH_CHECK(
state_cache.device().is_cpu() && gather_slots.device().is_cpu() && positions.device().is_cpu() &&
kv_slot_mapping.device().is_cpu() && rms_norm_weight.device().is_cpu() &&
cos_sin_cache.device().is_cpu() && kv_cache_2d.device().is_cpu(),
"compress_norm_rope_store_cpu: all inputs must be CPU tensors");
const int64_t window = gather_slots.size(1);
const int64_t state_width = state_cache.size(2) / 2;
const int64_t state_block_size = state_cache.size(1);
// Row strides are threaded through explicitly rather than assumed, since
// both buffers may be pages of a larger pooled allocation (matches
// store_cache.cpp/flash_mla.cpp's precedent).
const int64_t state_block_stride = state_cache.stride(0);
const int64_t cache_row_stride = kv_cache_2d.stride(0);
compress_norm_rope_store_impl(
state_cache.data_ptr<float>(),
state_block_stride,
state_block_size,
gather_slots.data_ptr<int64_t>(),
positions.data_ptr<int64_t>(),
kv_slot_mapping.data_ptr<int64_t>(),
rms_norm_weight.data_ptr<float>(),
static_cast<float>(rms_norm_eps),
cos_sin_cache.data_ptr<float>(),
kv_cache_2d.data_ptr<uint8_t>(),
cache_row_stride,
kv_cache_block_size,
compress_ratio,
window,
state_width,
num_tokens);
}
void compress_norm_rope_store_indexer_cpu(
at::Tensor& state_cache,
at::Tensor& gather_slots,
at::Tensor& positions,
at::Tensor& kv_slot_mapping,
at::Tensor& rms_norm_weight,
double rms_norm_eps,
at::Tensor& cos_sin_cache,
at::Tensor& kv_cache_2d,
int64_t kv_cache_block_size,
int64_t compress_ratio) {
TORCH_CHECK(state_cache.dim() == 3 && state_cache.scalar_type() == at::kFloat,
"compress_norm_rope_store_indexer_cpu: state_cache must be a 3D fp32 tensor");
TORCH_CHECK(state_cache.stride(2) == 1 && state_cache.stride(1) == state_cache.size(2),
"compress_norm_rope_store_indexer_cpu: state_cache rows must be contiguous within a block");
TORCH_CHECK(gather_slots.dim() == 2 && gather_slots.scalar_type() == at::kLong,
"compress_norm_rope_store_indexer_cpu: gather_slots must be a 2D int64 tensor");
const int64_t num_tokens = gather_slots.size(0);
TORCH_CHECK(positions.scalar_type() == at::kLong && positions.numel() == num_tokens,
"compress_norm_rope_store_indexer_cpu: positions must be int64 with num_tokens entries");
TORCH_CHECK(kv_slot_mapping.scalar_type() == at::kLong && kv_slot_mapping.numel() == num_tokens,
"compress_norm_rope_store_indexer_cpu: kv_slot_mapping must be int64 with num_tokens entries");
TORCH_CHECK(rms_norm_weight.numel() == kIndexerHeadDim,
"compress_norm_rope_store_indexer_cpu: rms_norm_weight must have head_dim (128) entries");
TORCH_CHECK(kv_cache_2d.dim() == 2 && kv_cache_2d.scalar_type() == at::kByte,
"compress_norm_rope_store_indexer_cpu: kv_cache_2d must be a 2D uint8 tensor");
TORCH_CHECK(state_cache.size(2) % 2 == 0,
"compress_norm_rope_store_indexer_cpu: state_cache width must be even");
TORCH_CHECK(
rms_norm_weight.scalar_type() == at::kFloat && rms_norm_weight.is_contiguous() &&
cos_sin_cache.scalar_type() == at::kFloat && cos_sin_cache.is_contiguous(),
"compress_norm_rope_store_indexer_cpu: rms_norm_weight/cos_sin_cache must "
"be float32 and contiguous (convert once in Python)");
TORCH_CHECK(
gather_slots.is_contiguous() && positions.is_contiguous() && kv_slot_mapping.is_contiguous(),
"compress_norm_rope_store_indexer_cpu: gather_slots/positions/kv_slot_mapping must be contiguous");
TORCH_CHECK(
state_cache.device().is_cpu() && gather_slots.device().is_cpu() && positions.device().is_cpu() &&
kv_slot_mapping.device().is_cpu() && rms_norm_weight.device().is_cpu() &&
cos_sin_cache.device().is_cpu() && kv_cache_2d.device().is_cpu(),
"compress_norm_rope_store_indexer_cpu: all inputs must be CPU tensors");
const int64_t window = gather_slots.size(1);
const int64_t state_width = state_cache.size(2) / 2;
const int64_t state_block_size = state_cache.size(1);
const int64_t state_block_stride = state_cache.stride(0);
const int64_t cache_row_stride = kv_cache_2d.stride(0);
compress_norm_rope_store_indexer_impl(
state_cache.data_ptr<float>(),
state_block_stride,
state_block_size,
gather_slots.data_ptr<int64_t>(),
positions.data_ptr<int64_t>(),
kv_slot_mapping.data_ptr<int64_t>(),
rms_norm_weight.data_ptr<float>(),
static_cast<float>(rms_norm_eps),
cos_sin_cache.data_ptr<float>(),
kv_cache_2d.data_ptr<uint8_t>(),
cache_row_stride,
kv_cache_block_size,
compress_ratio,
window,
state_width,
num_tokens);
}
+625
View File
@@ -0,0 +1,625 @@
// Adapted from
// https://github.com/sgl-project/sglang/blob/main/python/sglang/kernels/aot/csrc/cpu/flash_mla.cpp
//
// DeepSeek-V4 sparse MQA attention kernel for the fp8_ds_mla cache layout.
// Near-verbatim port of `sparse_mla_decode_kernel_impl` (VNNI K/V packing +
// AMX brgemm QK^T/PV + online softmax); the `V4_FP8Sparse` byte layout
// matches vLLM's fp8_ds_mla layout exactly (see store_cache.cpp).
//
// Differences from upstream:
// - Outer shape: upstream takes a padded batch-major query `[B, S_q, H, D]`
// with per-batch topk sets; vLLM passes flat `[num_tokens, H, D]` with
// per-token resolved indices. The kernel's own `at::parallel_for` already
// flattens `(batch, s_q)` internally, so this is called with
// `batches = num_tokens, s_q = 1` -- no padding, no kernel-body changes.
// - Two physically distinct paged caches (SWA "window" + compressor
// "compressed") are addressed via block_table-resolved flat slot ids,
// matching upstream's `k_main`/`k_extra` two-index-set convention.
// - `attn_sink` uses upstream's own post-hoc LSE correction
// (`out *= 1 / (1 + exp(sink - lse))`).
// - bf16 query only (matching upstream's own restriction: its FP8->bf16
// dequant intrinsics are bf16-specific); fp16 is not supported here.
//
// clang-format off
#include <algorithm>
#include <cmath>
#include <limits>
#include <vector>
#include "common.h"
#include "gemm.h"
#include "vec.h"
#if !defined(CPU_CAPABILITY_AVX512)
static_assert(false,
"flash_mla.cpp is only ever compiled into the AVX512+AMX-flagged "
"_C target (see cmake/cpu_extension.cmake's VLLM_EXT_SRC_SGL) and "
"relies on that unconditionally -- CPU_CAPABILITY_AVX512 must be "
"defined here.");
#endif
namespace {
constexpr int64_t kNopeDim = 448;
constexpr int64_t kRopeDim = 64;
constexpr int64_t kHeadDim = kNopeDim + kRopeDim;
constexpr int64_t kQuantBlock = 64;
constexpr int64_t kNumQuantBlocks = kNopeDim / kQuantBlock;
constexpr int64_t kScaleBytesPerToken = kNumQuantBlocks + 1;
constexpr int64_t kTokenDataBytes = kNopeDim + kRopeDim * 2;
// ---------------------------------------------------------------------------
// V4_FP8Sparse layout (SGLang's `flash_mla.cpp`) -- byte-for-byte identical
// to vLLM's own fp8_ds_mla layout above (d=512, d_nope=448, d_rope=64,
// tile=64, num_tiles=7):
// per block:
// [block_size * (d_nope + 2*d_rope) bytes ; FP8 NoPE + bf16 RoPE per token]
// [block_size * 8 bytes ; 7 e8m0 scales + 1 pad byte/token]
// ---------------------------------------------------------------------------
struct FP8LayoutMeta {
int64_t d;
int64_t d_nope;
int64_t d_rope;
int64_t tile_size;
int64_t num_tiles;
};
constexpr FP8LayoutMeta kV4Meta = {kHeadDim, kNopeDim, kRopeDim, kQuantBlock, kNumQuantBlocks};
// Convert one fp8_e8m0 byte (= unsigned 8-bit exponent) to float.
// e8m0 only stores an exponent (bias 127); value = 2^(e - 127), with 0xFF
// reserved for NaN.
inline float fp8_e8m0_to_float(uint8_t v) {
if (v == 0xFF) return std::numeric_limits<float>::quiet_NaN();
if (v == 0) return 0.f;
union {
uint32_t u;
float f;
} u;
u.u = static_cast<uint32_t>(v) << 23;
return u.f;
}
template <typename index_t>
inline bool is_valid_sparse_index(index_t idx, int64_t pos, int64_t topk_limit, int64_t total_tokens) {
const int64_t v = static_cast<int64_t>(idx);
return pos < topk_limit && v >= 0 && v < total_tokens;
}
#if defined(CPU_CAPABILITY_AVX512)
inline __attribute__((always_inline)) __m512i cvt_fp8_32_to_scaled_bf16(__m256i fp8, float scale) {
const __m512bh bf16_ext = CVT_FP8_TO_BF16_EXT(fp8);
const __m512 scale_vec = _mm512_mul_ps(_mm512_set1_ps(scale), _mm512_castsi512_ps(_mm512_set1_epi32(kFP8_BIAS)));
const __m512 f0 = _mm512_mul_ps(CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_ext, 0)), scale_vec);
const __m512 f1 = _mm512_mul_ps(CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_ext, 1)), scale_vec);
return (__m512i)_mm512_cvtne2ps_pbh(f1, f0);
}
// Load 32 contiguous logical dims (NoPE fp8-dequant-to-bf16, or raw bf16
// RoPE) starting at `dim_offset` from one token row of the V4_FP8Sparse
// layout.
inline __attribute__((always_inline)) __m512i load_fp8_kvcache_32_from_row(
const uint8_t* __restrict__ row_base, const uint8_t* __restrict__ scale_base, int dim_offset) {
if (dim_offset < kV4Meta.d_nope) {
const float scale = fp8_e8m0_to_float(scale_base[dim_offset / kV4Meta.tile_size]);
return cvt_fp8_32_to_scaled_bf16(
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(row_base + dim_offset)), scale);
}
const auto* rope_ptr = reinterpret_cast<const at::BFloat16*>(row_base + kV4Meta.d_nope);
return _mm512_loadu_si512(rope_ptr + dim_offset - kV4Meta.d_nope);
}
inline __attribute__((always_inline)) void init_fp8_kvcache_tile_rows(
const uint8_t* __restrict__ fp8_storage,
const int64_t block_size,
const int64_t storage_block_stride_bytes,
const int64_t token_idx,
const uint8_t*& row_base,
const uint8_t*& scale_base) {
constexpr int64_t nope_rope_per_token = kV4Meta.d_nope + 2 * kV4Meta.d_rope;
constexpr int64_t scale_stride = 8;
const int64_t block_idx = token_idx / block_size;
const int64_t block_off = token_idx - block_idx * block_size;
const uint8_t* block_base = fp8_storage + block_idx * storage_block_stride_bytes;
row_base = block_base + block_off * nope_rope_per_token;
scale_base = block_base + block_size * nope_rope_per_token + block_off * scale_stride;
}
template <bool convert_v, typename scalar_t, typename index_t, typename load_vec_t>
inline __attribute__((always_inline)) void sparse_pack_vnni_Nx32(
scalar_t* __restrict__ dst0,
scalar_t* __restrict__ dst1,
const index_t* __restrict__ ind,
const bool* __restrict__ valid_mask,
int N,
int dim_offset,
int ld_dst0,
int ld_dst1,
const load_vec_t& load_vec) {
__m512i vinputs[16];
int n = 0;
for (; n < N; ++n) {
if (!valid_mask[n]) {
vinputs[n] = _mm512_set1_epi32(0);
} else {
vinputs[n] = load_vec(n, static_cast<int64_t>(ind[n]), dim_offset);
}
}
for (; n < 16; ++n) {
vinputs[n] = _mm512_set1_epi32(0);
}
if constexpr (convert_v) {
for (int nn = 0; nn < 16; nn += 2) {
__m512i d0, d1;
std::tie(d0, d1) = transpose_2x32_16bit(vinputs[nn], vinputs[nn + 1]);
_mm512_storeu_si512(dst1 + (nn >> 1) * ld_dst1 * 2, d0);
_mm512_storeu_si512(dst1 + (nn >> 1) * ld_dst1 * 2 + 32, d1);
}
}
transpose_16x16_32bit(vinputs);
const __mmask16 vmask = (1 << N) - 1;
for (int k = 0; k < 16; ++k) {
_mm512_mask_storeu_epi32(dst0 + k * ld_dst0 * 2, vmask, vinputs[k]);
}
}
#endif // CPU_CAPABILITY_AVX512
// Gather+VNNI-pack `N` fp8-cache rows (dequantized to bf16 while packing)
// into `dst0` (K, vnni-packed for brgemm's B operand) and `dst1` (V,
// vnni-packed).
template <typename scalar_t, typename index_t>
void sparse_pack_vnni_fp8(
scalar_t* __restrict__ dst0,
scalar_t* __restrict__ dst1,
const uint8_t* __restrict__ fp8_storage,
const index_t* __restrict__ ind,
const bool* __restrict__ valid_mask,
int N,
int K,
int Kv,
int64_t block_size,
int64_t storage_block_stride_bytes,
int ld_dst0,
int ld_dst1) {
const int NB = div_up(N, 16);
const int KB = K / 32;
const int KBv = std::min(Kv / 32, KB);
for (int nb = 0; nb < NB; ++nb) {
const uint8_t* row_base[16];
const uint8_t* scale_base[16];
const int nb_size = std::min(N - nb * 16, 16);
for (int n = 0; n < nb_size; ++n) {
if (!valid_mask[nb * 16 + n]) {
row_base[n] = nullptr;
scale_base[n] = nullptr;
continue;
}
init_fp8_kvcache_tile_rows(
fp8_storage, block_size, storage_block_stride_bytes, static_cast<int64_t>(ind[nb * 16 + n]), row_base[n],
scale_base[n]);
}
auto load_vec = [&row_base, &scale_base](int n, int64_t idx, int dim_offset) {
UNUSED(idx);
return load_fp8_kvcache_32_from_row(row_base[n], scale_base[n], dim_offset);
};
for (int kb = 0; kb < KBv; ++kb) {
sparse_pack_vnni_Nx32<true, scalar_t, index_t>(
dst0 + ((kb * 32) >> 1) * ld_dst0 * 2 + nb * 16 * 2,
dst1 + ((nb * 16) >> 1) * ld_dst1 * 2 + kb * 32 * 2,
ind + nb * 16,
valid_mask + nb * 16,
nb_size,
kb * 32,
ld_dst0,
ld_dst1,
load_vec);
}
for (int kb = KBv; kb < KB; ++kb) {
sparse_pack_vnni_Nx32<false, scalar_t, index_t>(
dst0 + ((kb * 32) >> 1) * ld_dst0 * 2 + nb * 16 * 2,
dst1 + ((nb * 16) >> 1) * ld_dst1 * 2 + kb * 32 * 2,
ind + nb * 16,
valid_mask + nb * 16,
nb_size,
kb * 32,
ld_dst0,
ld_dst1,
load_vec);
}
}
}
template <typename scalar_t>
inline void fmla_fill_stub(scalar_t* __restrict__ out, float val, int64_t size) {
using Vec = at::vec::Vectorized<scalar_t>;
constexpr int kVecSize = Vec::size();
const Vec data_vec = Vec(static_cast<scalar_t>(val));
int64_t d = 0;
#pragma GCC unroll 4
for (; d <= size - kVecSize; d += kVecSize) {
data_vec.store(out + d);
}
if (size - d > 0) {
data_vec.store(out + d, size - d);
}
}
template <typename scalar_t, int BLOCK_N>
inline void fmla_copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input) {
static_assert(BLOCK_N % 32 == 0);
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
constexpr int COLS = BLOCK_N / 16;
auto store = [&](auto i) {
constexpr int col = i % COLS;
if constexpr (col % 2 == 0) {
fVec a0 = fVec::loadu(input + col * 16);
fVec a1 = fVec::loadu(input + col * 16 + 16);
bVec out_bvec = convert_from_float_ext<scalar_t>(a0, a1);
out_bvec.store(out + col * 16);
}
};
Unroll<COLS>{}(store);
}
template <typename scalar_t>
inline void fmla_finalize_out(scalar_t* __restrict__ out, const float* __restrict__ acc, float inv_s, int64_t size) {
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
constexpr int kVecSize = bVec::size();
const fVec s_fvec = fVec(inv_s);
int64_t d = 0;
#pragma GCC unroll 4
for (; d <= size - kVecSize; d += kVecSize) {
fVec a0 = fVec::loadu(acc + d) * s_fvec;
fVec a1 = fVec::loadu(acc + d + fVec::size()) * s_fvec;
bVec out_bvec = convert_from_float_ext<scalar_t>(a0, a1);
out_bvec.store(out + d);
}
for (; d < size; ++d) {
out[d] = static_cast<scalar_t>(acc[d] * inv_s);
}
}
// ---------------------------------------------------------------------------
// query : [num_tokens, H_q, D_qk] bf16 (SGLang's `[B, S_q, H_q, D]`
// with S_q folded away -- vLLM's flat per-token convention)
// indices : [num_tokens, topk_main] int64 (flat, block-table-resolved
// slot ids, -1 = invalid; SGLang's "main"/window index set)
// extra_* : same, for the optional "extra"/compressed index set
// attn_sink: [H_q] float32
// output : [num_tokens, H_q, D_v] bf16
// ---------------------------------------------------------------------------
template <typename scalar_t, typename index_t, int64_t BLOCK_N>
void sparse_mla_decode_kernel_impl(
scalar_t* __restrict__ output,
float* __restrict__ lse_out,
const scalar_t* __restrict__ query,
const uint8_t* __restrict__ k_main_fp8,
const uint8_t* __restrict__ k_extra_fp8,
const index_t* __restrict__ indices,
const index_t* __restrict__ extra_indices,
const float* __restrict__ attn_sink,
scalar_t* __restrict__ buffer,
int64_t batches,
int64_t s_q,
int64_t num_heads,
int64_t head_size,
int64_t head_size_v,
int64_t topk_main,
int64_t topk_extra,
int64_t total_tokens_main,
int64_t total_tokens_extra,
int64_t q_strideB,
int64_t q_strideH,
int64_t k_main_block_size,
int64_t k_extra_block_size,
int64_t k_main_block_stride_bytes,
int64_t k_extra_block_stride_bytes,
int64_t idx_strideB,
int64_t extra_idx_strideB,
float scaling,
int64_t buffer_size_per_thread) {
using Vec = at::vec::Vectorized<float>;
// partition heads
constexpr int64_t kBLOCK_H_MAX = 16;
const int64_t BLOCK_H = (batches * s_q) >= 16 ? kBLOCK_H_MAX : 8;
const int64_t num_h_blocks = div_up(num_heads, BLOCK_H);
// parallel on [B, S_q, head_block] -- with s_q == 1 (vLLM's flat call
// convention) this collapses to a plain [num_tokens, head_block] loop,
// one flat iteration per token with no padding to skip.
at::parallel_for(0, batches * s_q * num_h_blocks, 0, [&](int64_t begin, int64_t end) {
int64_t bs{0}, sq{0}, hb{0};
data_index_init(begin, bs, batches, sq, s_q, hb, num_h_blocks);
int tid = at::get_thread_num();
scalar_t* __restrict__ Btmp0 = buffer + tid * buffer_size_per_thread; // K packed
scalar_t* __restrict__ Btmp1 = Btmp0 + BLOCK_N * head_size; // V packed
// f32 V accumulator follows the bf16 packing region (reinterpret cast).
float* __restrict__ v_acc_local = reinterpret_cast<float*>(Btmp1 + BLOCK_N * head_size_v);
fmla_fill_stub(Btmp1, 0.f, BLOCK_N * head_size_v); // initialize V padding
alignas(64) float s_i[kBLOCK_H_MAX * BLOCK_N];
float* __restrict__ s_delta = s_i;
alignas(64) scalar_t s_delta2[kBLOCK_H_MAX * BLOCK_N];
alignas(64) float s_prime[kBLOCK_H_MAX];
alignas(64) float m_prime[kBLOCK_H_MAX];
for (int64_t i = begin; i < end; ++i) {
const int64_t h_start = hb * BLOCK_H;
const int64_t h_end = std::min(h_start + BLOCK_H, num_heads);
const int64_t h_size = h_end - h_start;
const scalar_t* __restrict__ q_ptr = query + bs * q_strideB + h_start * q_strideH;
const index_t* __restrict__ idx_ptr = indices + bs * idx_strideB;
const index_t* __restrict__ extra_idx_ptr = extra_indices == nullptr ? nullptr : extra_indices + bs * extra_idx_strideB;
fmla_fill_stub(s_prime, 0.f, BLOCK_H);
fmla_fill_stub(m_prime, -std::numeric_limits<float>::infinity(), BLOCK_H);
for (int64_t h = 0; h < h_size; ++h) {
fmla_fill_stub(v_acc_local + h * head_size_v, 0.f, head_size_v);
}
auto process_cache = [&](const uint8_t* __restrict__ fp8_ptr,
const index_t* __restrict__ cur_idx_ptr,
int64_t topk_count,
int64_t total_tokens,
int64_t fp8_block_size,
int64_t fp8_block_stride_bytes) {
if (fp8_ptr == nullptr || cur_idx_ptr == nullptr || topk_count == 0) {
return;
}
const int64_t topk_limit = topk_count;
for (int64_t n = 0; n < topk_limit; n += BLOCK_N) {
int64_t n_size = std::min<int64_t>(BLOCK_N, topk_limit - n);
const int64_t padded_n_size = div_up(int(n_size), TILE_K) * TILE_K;
bool valid_mask[BLOCK_N];
bool has_valid = false;
for (int64_t k = 0; k < n_size; ++k) {
const bool valid = is_valid_sparse_index(cur_idx_ptr[n + k], n + k, topk_limit, total_tokens);
valid_mask[k] = valid;
has_valid |= valid;
}
if (!has_valid) {
continue;
}
// Pack K (BLOCK_N rows via gather) into Btmp0 (key, vnni) and Btmp1
// (value, vnni), dequantizing FP8 cache entries while packing.
sparse_pack_vnni_fp8<scalar_t, index_t>(
Btmp0, Btmp1, fp8_ptr, cur_idx_ptr + n, valid_mask, static_cast<int>(n_size), static_cast<int>(head_size),
static_cast<int>(head_size_v), fp8_block_size, fp8_block_stride_bytes, static_cast<int>(BLOCK_N),
static_cast<int>(head_size_v));
// Q @ K
at::native::cpublas::brgemm(
/* M */ h_size,
/* N */ n_size,
/* K */ head_size,
/* lda */ q_strideH,
/* ldb */ BLOCK_N,
/* ldc */ BLOCK_N,
/* add_C */ false,
/* A */ q_ptr,
/* B */ Btmp0,
/* C */ s_i);
const Vec scale_vec = Vec(scaling);
for (int64_t h = 0; h < h_size; ++h) {
float* row = s_i + h * BLOCK_N;
at::vec::map<float>([scale_vec](Vec x) { return x * scale_vec; }, row, row, n_size);
for (int64_t k = 0; k < n_size; ++k) {
if (!valid_mask[k]) {
row[k] = -std::numeric_limits<float>::infinity();
}
}
// online softmax update
float m_i = at::vec::reduce_all<float>([](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, row, n_size);
m_i = std::max(m_i, m_prime[h]);
if (!std::isfinite(m_i)) {
fmla_fill_stub(s_delta + h * BLOCK_N, 0.f, padded_n_size);
fmla_copy_stub<scalar_t, BLOCK_N>(s_delta2 + h * BLOCK_N, s_delta + h * BLOCK_N);
continue;
}
const float m_delta = std::exp(m_prime[h] - m_i);
at::vec::map<float>([m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + h * BLOCK_N, row, n_size);
s_prime[h] *= m_delta;
s_prime[h] += at::vec::reduce_all<float>([](Vec& x, Vec& y) { return x + y; }, s_delta + h * BLOCK_N, n_size);
m_prime[h] = m_i;
float scale_m = m_delta;
at::vec::map<float>(
[scale_m](Vec x) { return x * Vec(scale_m); }, v_acc_local + h * head_size_v, v_acc_local + h * head_size_v,
head_size_v);
fmla_fill_stub(s_delta + h * BLOCK_N + n_size, 0.f, padded_n_size - n_size);
fmla_copy_stub<scalar_t, BLOCK_N>(s_delta2 + h * BLOCK_N, s_delta + h * BLOCK_N);
}
// V' <- s_delta @ V + V' (accumulate into v_acc_local at f32)
at::native::cpublas::brgemm(
/* M */ h_size,
/* N */ head_size_v,
/* K */ padded_n_size,
/* lda */ BLOCK_N,
/* ldb */ head_size_v,
/* ldc */ head_size_v,
/* add_C */ true,
/* A */ s_delta2,
/* B */ Btmp1,
/* C */ v_acc_local);
}
};
process_cache(k_main_fp8, idx_ptr, topk_main, total_tokens_main, k_main_block_size, k_main_block_stride_bytes);
process_cache(
k_extra_fp8, extra_idx_ptr, topk_extra, total_tokens_extra, k_extra_block_size, k_extra_block_stride_bytes);
// Apply attention sink correction directly on the output and lse.
// out *= exp(lse_no_sink) / (exp(lse_no_sink) + exp(attn_sink))
// = 1 / (1 + exp(attn_sink - lse_no_sink))
// (Algebraically equivalent to treating attn_sink as a virtual,
// zero-value extra key inside the softmax.)
for (int64_t h = 0; h < h_size; ++h) {
const int64_t hh = h_start + h;
const bool lonely = !std::isfinite(m_prime[h]) || s_prime[h] == 0.f;
float lse_val = lonely ? std::numeric_limits<float>::infinity() : (m_prime[h] + std::log(s_prime[h]));
float inv_s = lonely ? 0.f : (1.f / s_prime[h]);
if (!lonely && attn_sink != nullptr) {
const float sink = attn_sink[hh];
const float corr = 1.f / (1.f + std::exp(sink - lse_val));
inv_s *= corr;
}
scalar_t* out_row = output + bs * (s_q * num_heads * head_size_v) + sq * (num_heads * head_size_v) + hh * head_size_v;
if (lonely) {
fmla_fill_stub(out_row, 0.f, head_size_v);
} else {
fmla_finalize_out<scalar_t>(out_row, v_acc_local + h * head_size_v, inv_s, head_size_v);
}
lse_out[bs * num_heads * s_q + hh * s_q + sq] = lse_val;
}
data_index_step(bs, batches, sq, s_q, hb, num_h_blocks);
}
at::native::cpublas::brgemm_release();
});
}
} // anonymous namespace
void flash_mla_with_kvcache_cpu(
at::Tensor& out,
at::Tensor& q,
at::Tensor& window_cache_2d,
at::Tensor& window_slots,
int64_t window_block_size,
at::Tensor& compressed_cache_2d,
at::Tensor& compressed_slots,
int64_t compressed_block_size,
at::Tensor& attn_sink,
double scale) {
TORCH_CHECK(
q.dim() == 3 && q.size(2) == kHeadDim,
"flash_mla_with_kvcache_cpu: q must be [num_tokens, num_heads, 512]");
TORCH_CHECK(
out.dim() == 3 && out.size(0) == q.size(0) && out.size(1) == q.size(1) && out.size(2) == kHeadDim,
"flash_mla_with_kvcache_cpu: out must be [num_tokens, num_heads, 512]");
TORCH_CHECK(
out.scalar_type() == at::kBFloat16 && out.is_contiguous(),
"flash_mla_with_kvcache_cpu: out must be bf16 and contiguous "
"(pre-allocate once in Python -- see DeepseekV4CPUAttention.forward_mqa)");
TORCH_CHECK(out.device().is_cpu(), "flash_mla_with_kvcache_cpu: out must be a CPU tensor");
TORCH_CHECK(
q.scalar_type() == at::kBFloat16,
"flash_mla_with_kvcache_cpu: q must be bf16 (the AMX/VNNI kernel's "
"FP8->bf16 dequant intrinsics are bf16-specific)");
TORCH_CHECK(
window_slots.dim() == 2 && window_slots.size(0) == q.size(0),
"flash_mla_with_kvcache_cpu: window_slots must be [num_tokens, num_window]");
TORCH_CHECK(
compressed_slots.dim() == 2 && compressed_slots.size(0) == q.size(0),
"flash_mla_with_kvcache_cpu: compressed_slots must be [num_tokens, num_compressed]");
TORCH_CHECK(
window_slots.scalar_type() == at::kLong && compressed_slots.scalar_type() == at::kLong,
"flash_mla_with_kvcache_cpu: slot tensors must be int64");
TORCH_CHECK(
attn_sink.numel() == q.size(1), "flash_mla_with_kvcache_cpu: attn_sink must have num_heads entries");
TORCH_CHECK(
attn_sink.scalar_type() == at::kFloat && attn_sink.is_contiguous(),
"flash_mla_with_kvcache_cpu: attn_sink must be float32 and contiguous "
"(convert once in Python -- see DeepseekV4CPUAttention.forward_mqa)");
TORCH_CHECK(
window_slots.is_contiguous() && compressed_slots.is_contiguous(),
"flash_mla_with_kvcache_cpu: window_slots/compressed_slots must be contiguous");
TORCH_CHECK(
q.device().is_cpu() && window_cache_2d.device().is_cpu() && window_slots.device().is_cpu() &&
compressed_cache_2d.device().is_cpu() && compressed_slots.device().is_cpu() &&
attn_sink.device().is_cpu(),
"flash_mla_with_kvcache_cpu: all inputs must be CPU tensors");
const int64_t num_tokens = q.size(0);
const int64_t num_heads = q.size(1);
const int64_t num_window = window_slots.size(1);
const int64_t num_compressed = compressed_slots.size(1);
TORCH_CHECK(num_window > 0, "flash_mla_with_kvcache_cpu: window_slots must have at least one column");
// Row stride is threaded through explicitly rather than assumed to equal
// block_size * (token_data_bytes + scale_bytes_per_token): a paged/pooled
// cache buffer's physical row stride is not guaranteed to match the
// logical per-block byte count (matches store_cache.cpp's precedent).
const int64_t window_cache_row_stride = window_cache_2d.stride(0);
const int64_t compressed_cache_row_stride = num_compressed > 0 ? compressed_cache_2d.stride(0) : 0;
// AMX/VNNI path: flatten (batch, s_q) -> batches = num_tokens, s_q = 1.
CHECK_LAST_DIM_CONTIGUOUS_INPUT(q);
constexpr int64_t BLOCK_N = 128; // multiple of 32 for AMX brgemm
constexpr int64_t kBLOCK_H_MAX = 16;
static_assert(kHeadDim % 32 == 0, "head_dim must be a multiple of 32");
const int num_threads = at::get_num_threads();
// Layout per thread (in bf16 elements):
// [Btmp0 : BLOCK_N * D] [Btmp1 : BLOCK_N * D] [v_acc_local : kBLOCK_H_MAX * D floats]
// f32 takes 2 bf16 elements -> multiply by 2.
const int64_t buffer_size_per_thread = BLOCK_N * kHeadDim + BLOCK_N * kHeadDim + 2 * kBLOCK_H_MAX * kHeadDim;
auto buffer = at::empty({num_threads, buffer_size_per_thread}, q.options());
// lse is an internal scratch (attn_sink correction needs it per-token/head)
// -- not surfaced to the Python call site, which only consumes `out`.
auto lse = at::empty({num_tokens, num_heads}, q.options().dtype(at::kFloat));
const int64_t q_strideB = q.stride(0);
const int64_t q_strideH = q.stride(1);
const int64_t idx_strideB = window_slots.stride(0);
const int64_t extra_idx_strideB = num_compressed > 0 ? compressed_slots.stride(0) : 0;
const int64_t total_tokens_main = window_cache_2d.size(0) * window_block_size;
const int64_t total_tokens_extra = num_compressed > 0 ? compressed_cache_2d.size(0) * compressed_block_size : 0;
sparse_mla_decode_kernel_impl<at::BFloat16, int64_t, BLOCK_N>(
out.data_ptr<at::BFloat16>(),
lse.data_ptr<float>(),
q.data_ptr<at::BFloat16>(),
window_cache_2d.data_ptr<uint8_t>(),
num_compressed > 0 ? compressed_cache_2d.data_ptr<uint8_t>() : nullptr,
window_slots.data_ptr<int64_t>(),
num_compressed > 0 ? compressed_slots.data_ptr<int64_t>() : nullptr,
attn_sink.data_ptr<float>(),
buffer.data_ptr<at::BFloat16>(),
/* batches */ num_tokens,
/* s_q */ 1,
num_heads,
/* head_size */ kHeadDim,
/* head_size_v */ kHeadDim,
/* topk_main */ num_window,
/* topk_extra */ num_compressed,
total_tokens_main,
total_tokens_extra,
q_strideB,
q_strideH,
window_block_size,
num_compressed > 0 ? compressed_block_size : 1,
window_cache_row_stride,
compressed_cache_row_stride,
idx_strideB,
extra_idx_strideB,
static_cast<float>(scale),
buffer_size_per_thread);
}
+276
View File
@@ -0,0 +1,276 @@
// vLLM-native: DeepSeek-V4 sparse indexer Q-side RoPE + FP8 quant kernel.
// Ports the math of vLLM's own GPU/Triton reference
// (`_fused_indexer_q_rope_quant_kernel`, common/ops/fused_indexer_q.py) --
// not a port of SGLang's own indexer-Q kernel, since that applies RoPE plus
// a 128-point Hadamard rotation the real DeepSeek-V4-Flash checkpoint
// doesn't use (verified: it applies RoPE only, no Hadamard step).
//
// Per (token, head): GPT-J interleaved RoPE on the last `rot_dim` dims;
// leading NoPE dims pass through unrotated. RoPE-rotated values are bf16-
// round-tripped before the absmax reduction (matching the K-side compressor
// kernel), NoPE values are not. One UE8M0 scale per (token, head) covers the
// full head_dim; since the FP8 Q tensor has no companion scale tensor, the
// scale is folded into `index_weights_out` instead:
// weights_out = weights * q_scale * softmax_scale * head_scale
// MXFP4 (`use_fp4=True`) has per-block scales that can't be folded this way
// and stays on triton-cpu -- out of scope here.
//
// AVX512 path: NoPE copy+absmax and the FP8 scale/clamp use plain 16-wide
// vector ops (final FP8 cast stays scalar, matching SGLang's own
// `act_quant_cpu_impl`). The RoPE rotation vectorizes the interleaved
// (x_even, x_odd) pairs via in-register permutes (`_mm512_permute_ps` /
// `_mm512_permutexvar_ps`), the same technique compressor.cpp's
// `apply_gptj_rope_inplace` and store_cache.cpp's
// `inverse_gptj_rope_o_proj_cpu` use, since gather/scatter is comparatively
// expensive on AVX512.
#include <algorithm>
#include <cmath>
#include <vector>
#include "common.h"
#include "vec.h"
#if !defined(CPU_CAPABILITY_AVX512)
static_assert(
false,
"indexer.cpp is only ever compiled into the AVX512+AMX-flagged "
"_C target (see cmake/cpu_extension.cmake's VLLM_EXT_SRC_SGL) and "
"relies on that unconditionally -- CPU_CAPABILITY_AVX512 must be "
"defined here.");
#endif
namespace {
constexpr float kFp8Max = 448.0f;
#if defined(CPU_CAPABILITY_AVX512)
// fp32 -> bf16 -> fp32 round trip via the AVX512-BF16 conversion
// instruction, numerically matching `static_cast<float>(at::BFloat16(x))`
// (round-to-nearest-even).
inline __m512 bf16_round_trip_avx512(__m512 x) {
return CVT_BF16_TO_FP32((__m256i)(_mm512_cvtneps_pbh(x)));
}
inline __m512 abs_ps(__m512 x) {
return _mm512_andnot_ps(_mm512_set1_ps(-0.0f), x);
}
#endif // CPU_CAPABILITY_AVX512
void fused_indexer_q_rope_quant_impl(
const int64_t* __restrict__ positions, // [num_tokens]
const float* __restrict__ index_q, // [num_tokens, num_heads, head_dim]
int64_t index_q_stride0, int64_t index_q_stride1,
const float* __restrict__ cos_sin_cache, // [max_pos, rot_dim]
int64_t cos_sin_stride, int64_t half_rot_dim,
at::Float8_e4m3fn* __restrict__ index_q_fp8, // [num_tokens, num_heads,
// head_dim]
int64_t index_q_fp8_stride0, int64_t index_q_fp8_stride1, int64_t head_dim,
const float* __restrict__ index_weights, // [num_tokens, num_heads]
int64_t index_weights_stride, float softmax_scale, float head_scale,
float* __restrict__ index_weights_out, // [num_tokens, num_heads]
int64_t index_weights_out_stride, int64_t num_tokens, int64_t num_heads) {
const int64_t rot_dim = 2 * half_rot_dim;
const int64_t nope_dim = head_dim - rot_dim;
at::parallel_for(
0, num_tokens * num_heads, 0, [&](int64_t begin, int64_t end) {
std::vector<float> buf(head_dim);
for (int64_t idx = begin; idx < end; ++idx) {
const int64_t t = idx / num_heads;
const int64_t h = idx % num_heads;
const int64_t pos = positions[t];
const float* const cos = cos_sin_cache + pos * cos_sin_stride;
const float* const sin = cos + half_rot_dim;
const float* const q =
index_q + t * index_q_stride0 + h * index_q_stride1;
float amax = 1e-4f;
#if defined(CPU_CAPABILITY_AVX512)
{
__m512 vamax = _mm512_set1_ps(amax);
for (int64_t d = 0; d < nope_dim; d += 16) {
__m512 v = _mm512_loadu_ps(q + d);
_mm512_storeu_ps(&buf[d], v);
vamax = _mm512_max_ps(vamax, abs_ps(v));
}
amax = _mm512_reduce_max_ps(vamax);
}
#endif
const float* const q_rot = q + nope_dim;
#if defined(CPU_CAPABILITY_AVX512)
{
// Permute-based (no gather/scatter), same technique as
// compressor.cpp's apply_gptj_rope_inplace / store_cache.cpp's
// inverse_gptj_rope_o_proj_cpu: `_mm512_permute_ps` swaps each
// pair's (even, odd) lanes in one shuffle, and cos/sin -- loaded as
// 8 contiguous values -- are duplicated pairwise via
// `_mm512_permutexvar_ps` to align with that layout. Two batches of
// 8 pairs per iteration (16 total, matching the old gather step).
// Unlike the RoPE kernels above, the result here stays in its
// original interleaved layout, so no un-permute/scatter is needed
// before the bf16 round-trip and store.
const __m512i dup_idx = _mm512_set_epi32(7, 7, 6, 6, 5, 5, 4, 4, 3,
3, 2, 2, 1, 1, 0, 0);
// Forward GPT-J rotation negates the sin broadcast on the even
// (real) lane of each pair; the odd (imag) lane keeps sin's sign.
const __m512 sign_mask = _mm512_castsi512_ps(_mm512_set_epi32(
0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0,
(int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0,
(int)0x80000000, 0, (int)0x80000000));
__m512 vamax_rope = _mm512_setzero_ps();
for (int64_t i = 0; i < half_rot_dim; i += 16) {
const float* const base = q_rot + 2 * i;
float* const dst = buf.data() + nope_dim + 2 * i;
__m512 xv0 = _mm512_loadu_ps(base);
__m512 cos_b0 = _mm512_permutexvar_ps(
dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(cos + i)));
__m512 sin_b0 = _mm512_permutexvar_ps(
dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(sin + i)));
__m512 out0 = _mm512_fmadd_ps(
xv0, cos_b0,
_mm512_mul_ps(_mm512_permute_ps(xv0, 0xB1),
_mm512_xor_ps(sin_b0, sign_mask)));
// Match reference numerics: fp32 -> bf16 -> fp32 before the
// absmax.
out0 = bf16_round_trip_avx512(out0);
_mm512_storeu_ps(dst, out0);
vamax_rope = _mm512_max_ps(vamax_rope, abs_ps(out0));
__m512 xv1 = _mm512_loadu_ps(base + 16);
__m512 cos_b1 = _mm512_permutexvar_ps(
dup_idx,
_mm512_castps256_ps512(_mm256_loadu_ps(cos + i + 8)));
__m512 sin_b1 = _mm512_permutexvar_ps(
dup_idx,
_mm512_castps256_ps512(_mm256_loadu_ps(sin + i + 8)));
__m512 out1 = _mm512_fmadd_ps(
xv1, cos_b1,
_mm512_mul_ps(_mm512_permute_ps(xv1, 0xB1),
_mm512_xor_ps(sin_b1, sign_mask)));
out1 = bf16_round_trip_avx512(out1);
_mm512_storeu_ps(dst + 16, out1);
vamax_rope = _mm512_max_ps(vamax_rope, abs_ps(out1));
}
amax = std::max(amax, _mm512_reduce_max_ps(vamax_rope));
}
#endif
const float exponent = std::ceil(std::log2(amax / kFp8Max));
const float scale = std::exp2(exponent);
const float inv_scale = 1.0f / scale;
at::Float8_e4m3fn* const out =
index_q_fp8 + t * index_q_fp8_stride0 + h * index_q_fp8_stride1;
#if defined(CPU_CAPABILITY_AVX512)
{
const __m512 vinv = _mm512_set1_ps(inv_scale);
const __m512 vmax = _mm512_set1_ps(kFp8Max);
const __m512 vmin = _mm512_set1_ps(-kFp8Max);
alignas(64) float scaled_buf[16];
for (int64_t d2 = 0; d2 < head_dim; d2 += 16) {
__m512 v = _mm512_loadu_ps(&buf[d2]);
v = _mm512_min_ps(_mm512_max_ps(_mm512_mul_ps(v, vinv), vmin),
vmax);
_mm512_store_ps(scaled_buf, v);
for (int64_t j = 0; j < 16; ++j)
out[d2 + j] = at::Float8_e4m3fn(scaled_buf[j]);
}
}
#endif
float w = index_weights[t * index_weights_stride + h];
w *= scale;
w *= softmax_scale;
w *= head_scale;
index_weights_out[t * index_weights_out_stride + h] = w;
}
});
}
} // anonymous namespace
void fused_indexer_q_rope_quant_cpu(at::Tensor& positions, at::Tensor& index_q,
at::Tensor& index_q_cos_sin_cache,
at::Tensor& index_q_fp8,
at::Tensor& index_weights,
double index_weights_softmax_scale,
double index_weights_head_scale,
at::Tensor& index_weights_out) {
TORCH_CHECK(
positions.dim() == 1 && positions.scalar_type() == at::kLong,
"fused_indexer_q_rope_quant_cpu: positions must be a 1D int64 tensor");
TORCH_CHECK(positions.is_contiguous(),
"fused_indexer_q_rope_quant_cpu: positions must be contiguous");
TORCH_CHECK(index_q.dim() == 3,
"fused_indexer_q_rope_quant_cpu: index_q must be 3D");
TORCH_CHECK(index_q.scalar_type() == at::kFloat,
"fused_indexer_q_rope_quant_cpu: index_q must be float32 "
"(convert once in Python)");
TORCH_CHECK(index_q.stride(2) == 1,
"fused_indexer_q_rope_quant_cpu: index_q must be contiguous in "
"the head_dim axis");
TORCH_CHECK(
index_q_cos_sin_cache.dim() == 2 && index_q_cos_sin_cache.stride(1) == 1,
"fused_indexer_q_rope_quant_cpu: index_q_cos_sin_cache must be a "
"contiguous 2D tensor");
TORCH_CHECK(
index_q_cos_sin_cache.scalar_type() == at::kFloat,
"fused_indexer_q_rope_quant_cpu: index_q_cos_sin_cache must be float32 "
"(convert once in Python)");
TORCH_CHECK(index_q_fp8.sizes() == index_q.sizes() &&
index_q_fp8.scalar_type() == at::kFloat8_e4m3fn &&
index_q_fp8.stride(2) == 1,
"fused_indexer_q_rope_quant_cpu: index_q_fp8 must match "
"index_q's shape, be "
"float8_e4m3fn, and contiguous in the head_dim axis");
TORCH_CHECK(index_weights.dim() == 2 && index_weights.stride(1) == 1,
"fused_indexer_q_rope_quant_cpu: index_weights must be a "
"contiguous-in-head 2D tensor");
TORCH_CHECK(index_weights.scalar_type() == at::kFloat,
"fused_indexer_q_rope_quant_cpu: index_weights must be float32 "
"(convert once in Python)");
TORCH_CHECK(index_weights_out.sizes() == index_weights.sizes() &&
index_weights_out.scalar_type() == at::kFloat &&
index_weights_out.stride(1) == 1,
"fused_indexer_q_rope_quant_cpu: index_weights_out must match "
"index_weights' shape, "
"be float32, and contiguous in the head axis");
TORCH_CHECK(positions.device().is_cpu() && index_q.device().is_cpu() &&
index_q_cos_sin_cache.device().is_cpu() &&
index_q_fp8.device().is_cpu() &&
index_weights.device().is_cpu() &&
index_weights_out.device().is_cpu(),
"fused_indexer_q_rope_quant_cpu: all inputs must be CPU tensors");
const int64_t num_tokens = positions.size(0);
const int64_t num_heads = index_q.size(1);
const int64_t head_dim = index_q.size(2);
const int64_t half_rot_dim = index_q_cos_sin_cache.size(-1) / 2;
// Only ever called with head_dim=128, rot_dim=64 (DeepseekV4's sparse
// indexer Q side); the AVX512 loops below process 16 elements/pairs per
// iteration with no scalar remainder, so require even divisibility.
TORCH_CHECK(head_dim % 16 == 0 && (head_dim - 2 * half_rot_dim) % 16 == 0 &&
half_rot_dim % 16 == 0,
"fused_indexer_q_rope_quant_cpu: head_dim, head_dim-rot_dim, and "
"rot_dim/2 "
"must be multiples of 16");
fused_indexer_q_rope_quant_impl(
positions.data_ptr<int64_t>(), index_q.data_ptr<float>(),
index_q.stride(0), index_q.stride(1),
index_q_cos_sin_cache.data_ptr<float>(), index_q_cos_sin_cache.stride(0),
half_rot_dim, index_q_fp8.data_ptr<at::Float8_e4m3fn>(),
index_q_fp8.stride(0), index_q_fp8.stride(1), head_dim,
index_weights.data_ptr<float>(), index_weights.stride(0),
static_cast<float>(index_weights_softmax_scale),
static_cast<float>(index_weights_head_scale),
index_weights_out.data_ptr<float>(), index_weights_out.stride(0),
num_tokens, num_heads);
}
File diff suppressed because it is too large Load Diff
+9 -9
View File
@@ -1,14 +1,14 @@
// vLLM-native CPU cache-write op for MLA's single-latent-buffer KV cache.
// `concat_and_cache_mla` (the generic op used by every GPU backend) is
// CUDA-only; this is the CPU counterpart, adapted in spirit from SGLang's
// `store_cache_cpu` (csrc/cpu/kvcache.cpp).
//
// `concat_and_cache_mla` (the generic MLA cache-write op used by every GPU
// backend) is registered CUDA-only. This is the CPU counterpart, adapted in
// spirit from SGLang's `store_cache_cpu` (csrc/cpu/kvcache.cpp) but
// generalized to write two source tensors (`kv_c_normed`, `k_pe`) into two
// different column-offset ranges of the SAME destination row -- SGLang's
// version assumes k/v land in two independent, equal-row-width cache
// tensors, which doesn't hold here since MLA's cache is one 576-wide buffer
// and the two column ranges (512-wide, 64-wide) don't match the buffer's
// true per-token stride, so the write can't reuse `store_cache_cpu` as-is.
// Difference from upstream: writes two source tensors (`kv_c_normed`,
// `k_pe`) into two column-offset ranges of the SAME destination row.
// Upstream assumes k/v land in two independent, equal-row-width cache
// tensors -- MLA's cache is one 576-wide buffer whose two column ranges
// (512-wide, 64-wide) don't match that per-token stride, so
// `store_cache_cpu` can't be reused as-is.
#include "common.h"
#include "vec.h"
+3 -4
View File
@@ -871,13 +871,13 @@ static inline void check_moe_scales(
// topk_ids: [M, topk] (int32_t)
//
at::Tensor fused_experts_cpu(
void fused_experts_cpu(
at::Tensor& out,
at::Tensor& hidden_states,
at::Tensor& w1,
at::Tensor& w2,
at::Tensor& topk_weights,
at::Tensor& topk_ids,
bool inplace,
int64_t moe_comp_method,
const std::optional<at::Tensor>& w1_scale,
const std::optional<at::Tensor>& w2_scale,
@@ -943,7 +943,7 @@ at::Tensor fused_experts_cpu(
// check scales
check_moe_scales(moe_comp_method, w1_scale, w2_scale, block_size);
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
at::Tensor& out_hidden_states = out;
// NB: worst case is each expert holds a block with remainder of 1
// 1. sorted_ids : [M * topk + E * (BLOCK_M - 1)]
@@ -1214,7 +1214,6 @@ at::Tensor fused_experts_cpu(
with_bias);
}
});
return out_hidden_states;
}
// shared expert kernel
+217
View File
@@ -0,0 +1,217 @@
// Adapted from
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu:
// `fp8_paged_mqa_logits_cpu`/`fp8_paged_mqa_logits_cpu_impl` (DECODE path).
//
// Reads the paged indexer K-cache directly via `page_table` -- no separate
// eager gather step -- and processes the whole decode batch (one query token
// per request; the CPU indexer decode path doesn't support native MTP/spec
// decode yet, see `sparse_attn_indexer_cpu`) in a single `at::parallel_for`
// flattening `(batch, token)`, exactly like upstream. AVX512 `dot_fp8_128`
// dequant-dot-product and the split K-region-then-scale-region page byte
// layout are verbatim (that part of the upstream contract already matches
// vLLM's cache, written by `compress_norm_rope_store_indexer_cpu` in
// compressor.cpp).
//
// Differences from upstream:
// - `block_size` is a runtime argument instead of the hardcoded
// `kBlockSize = 64`. vLLM's indexer K-cache shares the model's single
// global `--block-size` KV-cache-group setting (confirmed empirically:
// the DeepSeek-V4-Flash TP1/TP4 baseline configs run with
// `block_size=256`, not 64 -- see `.agent-workspace/perf_baseline_tp*_
// server.log`), so the upstream `kvcache_fp8.size(1) == 64` shape
// contract does not hold in general.
// - `q_fp8`/`weight` drop the redundant `S_q=1` dimension (`[batch, heads,
// 128]`/`[batch, heads]` instead of `[batch, 1, heads, 128]`) since S_q is
// always 1 here; `weight` is fp32-only (already folds q_scale/softmax_
// scale/head_scale, matching this call site's actual dtype -- unlike
// upstream's bf16/fp16/fp32 dispatch, which vLLM's CPU indexer never
// produces).
// - `index_head_dim` (128) stays a compile-time constant, matching
// upstream: it's `config.index_head_dim`, a genuine DeepSeek-V4 model
// invariant, not a vLLM KV-cache config knob like `block_size`.
#include "common.h"
#include "vec.h"
#if !defined(CPU_CAPABILITY_AVX512)
static_assert(false,
"paged_mqa_logits.cpp is only ever compiled into the "
"AVX512+AMX-flagged _C target (see cmake/cpu_extension.cmake's "
"VLLM_EXT_SRC_SGL) and relies on that unconditionally -- "
"CPU_CAPABILITY_AVX512 must be defined here.");
#endif
namespace {
constexpr int64_t kIndexHeadDim = 128;
constexpr int64_t kIndexHeadDimWithScaleBytes = 132;
inline float dot_fp8_128(const uint8_t* k, const uint8_t* q) {
__m512 acc = _mm512_setzero_ps();
for (int64_t d = 0; d < kIndexHeadDim; d += 32) {
const __m256i k8 =
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(k + d));
const __m256i q8 =
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(q + d));
acc = _mm512_dpbf16_ps(acc, CVT_FP8_TO_BF16(k8), CVT_FP8_TO_BF16(q8));
}
return _mm512_reduce_add_ps(acc);
}
template <typename seq_t, typename page_t>
void fp8_paged_mqa_logits_cpu_impl(const at::Tensor& q_fp8,
const at::Tensor& kvcache_fp8,
const at::Tensor& weight,
const at::Tensor& seq_lens,
const at::Tensor& page_table,
at::Tensor& logits, int64_t block_size,
int64_t max_seq_len) {
const int64_t batch_size = q_fp8.size(0);
const int64_t num_heads = q_fp8.size(1);
const int64_t num_pages = kvcache_fp8.size(0);
// Row stride is threaded through explicitly rather than assumed equal to
// the page byte width: the K-cache may be a page of a larger pooled
// allocation shared across layers (matches store_cache.cpp/flash_mla.cpp's
// precedent).
const int64_t page_stride = kvcache_fp8.stride(0);
const int64_t pages_per_batch = page_table.size(1);
const int64_t scale_offset_bytes = block_size * kIndexHeadDim;
const auto* q_ptr = reinterpret_cast<const uint8_t*>(q_fp8.const_data_ptr());
const auto* cache_ptr =
reinterpret_cast<const uint8_t*>(kvcache_fp8.const_data_ptr());
const auto* weight_ptr = weight.const_data_ptr<float>();
const auto* seq_ptr = seq_lens.const_data_ptr<seq_t>();
const auto* page_ptr = page_table.const_data_ptr<page_t>();
auto* out_ptr = logits.data_ptr<float>();
at::parallel_for(
0, batch_size * max_seq_len, GRAIN_SIZE / kIndexHeadDim,
[&](int64_t begin, int64_t end) {
int64_t b{0}, token{0};
data_index_init(begin, b, batch_size, token, max_seq_len);
for (int64_t i = begin; i < end; ++i) {
const int64_t seq_len = static_cast<int64_t>(seq_ptr[b]);
if (token >= seq_len) {
data_index_step(b, batch_size, token, max_seq_len);
continue;
}
const int64_t q_batch_offset = b * num_heads * kIndexHeadDim;
const int64_t weight_batch_offset = b * num_heads;
const int64_t page_batch_offset = b * pages_per_batch;
float* out_row = out_ptr + b * max_seq_len;
const int64_t logical_page = token / block_size;
const int64_t token_in_page = token % block_size;
const int64_t physical_page =
static_cast<int64_t>(page_ptr[page_batch_offset + logical_page]);
const uint8_t* block = cache_ptr + physical_page * page_stride;
const uint8_t* k_token = block + token_in_page * kIndexHeadDim;
const float* scale_ptr =
reinterpret_cast<const float*>(block + scale_offset_bytes);
const float k_scale = scale_ptr[token_in_page];
float score_sum = 0.0f;
for (int64_t h = 0; h < num_heads; ++h) {
const uint8_t* q_head = q_ptr + q_batch_offset + h * kIndexHeadDim;
float dot = dot_fp8_128(k_token, q_head);
dot = std::max(dot, 0.0f);
score_sum += dot * weight_ptr[weight_batch_offset + h];
}
out_row[token] = score_sum * k_scale;
data_index_step(b, batch_size, token, max_seq_len);
}
});
}
template <typename seq_t>
void dispatch_page_type(const at::Tensor& q_fp8, const at::Tensor& kvcache_fp8,
const at::Tensor& weight, const at::Tensor& seq_lens,
const at::Tensor& page_table, at::Tensor& logits,
int64_t block_size, int64_t max_seq_len) {
if (page_table.scalar_type() == at::kInt) {
fp8_paged_mqa_logits_cpu_impl<seq_t, int32_t>(q_fp8, kvcache_fp8, weight,
seq_lens, page_table, logits,
block_size, max_seq_len);
} else if (page_table.scalar_type() == at::kLong) {
fp8_paged_mqa_logits_cpu_impl<seq_t, int64_t>(q_fp8, kvcache_fp8, weight,
seq_lens, page_table, logits,
block_size, max_seq_len);
} else {
TORCH_CHECK(false, "page_table must be int32 or int64");
}
}
} // namespace
at::Tensor fp8_paged_mqa_logits_cpu(at::Tensor& q_fp8, at::Tensor& kvcache_fp8,
at::Tensor& weight, at::Tensor& seq_lens,
at::Tensor& page_table, int64_t block_size,
int64_t max_seq_len) {
CHECK_INPUT(q_fp8);
CHECK_CPU(kvcache_fp8);
CHECK_INPUT(weight);
CHECK_INPUT(seq_lens);
CHECK_INPUT(page_table);
TORCH_CHECK(
q_fp8.dim() == 3 && q_fp8.scalar_type() == at::kFloat8_e4m3fn,
"fp8_paged_mqa_logits_cpu: q_fp8 must be a 3D float8_e4m3fn tensor "
"[batch, heads, 128]");
TORCH_CHECK(q_fp8.size(2) == kIndexHeadDim,
"fp8_paged_mqa_logits_cpu: q_fp8 head_dim must be 128");
TORCH_CHECK(kvcache_fp8.dim() == 2 && kvcache_fp8.scalar_type() == at::kByte,
"fp8_paged_mqa_logits_cpu: kvcache_fp8 must be a 2D uint8 tensor "
"[num_pages, buf_numel_per_page]");
TORCH_CHECK(kvcache_fp8.stride(1) == 1 &&
kvcache_fp8.stride(0) >= kvcache_fp8.size(1),
"fp8_paged_mqa_logits_cpu: kvcache_fp8 must be contiguous "
"within a page (may be a page of a larger pooled allocation "
"between pages)");
TORCH_CHECK(block_size > 0,
"fp8_paged_mqa_logits_cpu: block_size must be positive");
TORCH_CHECK(
kvcache_fp8.size(1) >= block_size * kIndexHeadDimWithScaleBytes,
"fp8_paged_mqa_logits_cpu: kvcache_fp8 page byte width too small for "
"block_size");
const int64_t batch_size = q_fp8.size(0);
const int64_t num_heads = q_fp8.size(1);
TORCH_CHECK(
weight.dim() == 2 && weight.scalar_type() == at::kFloat &&
weight.size(0) == batch_size && weight.size(1) == num_heads,
"fp8_paged_mqa_logits_cpu: weight must be a 2D fp32 tensor [batch, "
"heads]");
TORCH_CHECK(seq_lens.dim() == 1 && seq_lens.size(0) == batch_size,
"fp8_paged_mqa_logits_cpu: seq_lens must be a 1D tensor [batch]");
TORCH_CHECK(page_table.dim() == 2 && page_table.size(0) == batch_size,
"fp8_paged_mqa_logits_cpu: page_table must be a 2D tensor "
"[batch, pages]");
TORCH_CHECK(max_seq_len >= 0,
"fp8_paged_mqa_logits_cpu: max_seq_len must be non-negative");
// Positions with token >= seq_len are never written by the kernel below,
// and topk_transform_512_cpu (the only consumer) never reads them either
// -- left uninitialized, matching upstream.
auto logits =
at::empty({batch_size, max_seq_len}, q_fp8.options().dtype(at::kFloat));
if (batch_size == 0 || max_seq_len == 0) {
return logits;
}
if (seq_lens.scalar_type() == at::kInt) {
dispatch_page_type<int32_t>(q_fp8, kvcache_fp8, weight, seq_lens,
page_table, logits, block_size, max_seq_len);
} else if (seq_lens.scalar_type() == at::kLong) {
dispatch_page_type<int64_t>(q_fp8, kvcache_fp8, weight, seq_lens,
page_table, logits, block_size, max_seq_len);
} else {
TORCH_CHECK(false, "seq_lens must be int32 or int64");
}
return logits;
}
+481
View File
@@ -0,0 +1,481 @@
// Adapted from
// https://github.com/sgl-project/sglang/blob/main/python/sglang/kernels/aot/csrc/cpu/store_cache.cpp
//
// DeepSeek-V4 fused Q RMSNorm+RoPE / KV RoPE+UE8M0-quant+paged-cache-insert
// kernel for the fp8_ds_mla cache layout. Not a port of upstream's
// `set_k_and_s_cpu`/`quant_to_nope_fp8_rope_bf16_pack_cpu` -- their
// pool_kv/pool_score signature doesn't apply to vLLM's layout. This is the
// CPU counterpart to vLLM's own CUDA kernel
// (`fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu`); see
// `DeepseekV4Attention._fused_qnorm_rope_kv_insert_cpu` (attention.py) for
// the eager reference this replaces.
//
// clang-format off
#include <algorithm>
#include <cmath>
#include <vector>
#include "common.h"
#include "vec.h"
#if !defined(CPU_CAPABILITY_AVX512)
static_assert(false,
"store_cache.cpp is only ever compiled into the AVX512+AMX-flagged "
"_C target (see cmake/cpu_extension.cmake's VLLM_EXT_SRC_SGL) and "
"relies on that unconditionally -- CPU_CAPABILITY_AVX512 must be "
"defined here.");
#endif
namespace {
constexpr int64_t kRopeDim = 64;
constexpr int64_t kQuantBlock = 64;
constexpr float kFp8Max = 448.0f;
constexpr float kFp8Min = -448.0f;
constexpr float kQuantEps = 1e-4f;
inline void apply_gptj_rope_inplace(float* __restrict__ x, const float* __restrict__ cos, const float* __restrict__ sin, int64_t half_dim) {
for (int64_t i = 0; i < half_dim; ++i) {
const float e = x[2 * i];
const float o = x[2 * i + 1];
x[2 * i] = e * cos[i] - o * sin[i];
x[2 * i + 1] = e * sin[i] + o * cos[i];
}
}
// AVX512 tile-quantization, adapted verbatim from SGLang's
// `store_cache.cpp` (`quant_to_nope_fp8_rope_bf16_pack_cpu`'s
// `cvt_fp32x16_to_fp8x16`/`quantize_tile_avx512`): per-64-element tile,
// computes the UE8M0 power-of-two scale and quantizes to fp8_e4m3fn with
// RNE rounding via bit manipulation instead of a per-element scalar
// `at::Float8_e4m3fn` cast. Subnormal-magnitude fp8 lanes are flushed to
// zero (matches SGLang's own convention, `cvt_fp32x16_to_fp8x16`) --
// vLLM's own AVX512 dequant of this cache (`flash_mla.cpp`'s
// `CVT_FP8_TO_BF16_EXT`) doesn't reconstruct exact subnormal magnitudes
// either way, so there is nothing to preserve by rounding them exactly on
// the write side.
#if defined(CPU_CAPABILITY_AVX512)
// Vectorized float32x16 -> fp8_e4m3fn x16 using AVX512 bit manipulation.
// Implements RNE (round-to-nearest-even). Flushes subnormal fp8 to zero.
// fp8_e4m3fn: sign(1), exp(4, bias=7), mant(3), no inf/nan special values.
// Normal: exp_field in [1..15], value = 2^(exp-7) * (1 + mant/8)
// Input assumed clamped to [-448, 448].
inline __m128i cvt_fp32x16_to_fp8x16(__m512 input) {
__m512i bits = _mm512_castps_si512(input);
// Extract sign -> bit 7 of fp8 byte
__m512i signs = _mm512_srli_epi32(_mm512_and_si512(bits, _mm512_set1_epi32(0x80000000)), 24);
// Absolute value bits
__m512i abs_bits = _mm512_and_si512(bits, _mm512_set1_epi32(0x7FFFFFFF));
// Float32 biased exponent
__m512i f32_exp = _mm512_srli_epi32(abs_bits, 23);
// Float32 mantissa (23 bits)
__m512i f32_mant = _mm512_and_si512(abs_bits, _mm512_set1_epi32(0x7FFFFF));
// fp8_exp = f32_exp - 120 (rebias from 127 to 7)
__m512i fp8_exp = _mm512_sub_epi32(f32_exp, _mm512_set1_epi32(120));
// Top 3 mantissa bits
__m512i mant3 = _mm512_srli_epi32(f32_mant, 20);
// RNE rounding: round_bit = bit 19, sticky = bits[18:0]
__m512i round_bit = _mm512_and_si512(_mm512_srli_epi32(f32_mant, 19), _mm512_set1_epi32(1));
__m512i sticky_bits = _mm512_and_si512(f32_mant, _mm512_set1_epi32(0x7FFFF));
__mmask16 has_sticky = _mm512_cmpneq_epi32_mask(sticky_bits, _mm512_setzero_si512());
// Round up if round_bit AND (sticky OR lsb_of_mant3)
__m512i lsb = _mm512_and_si512(mant3, _mm512_set1_epi32(1));
__m512i sticky_or_lsb = _mm512_or_si512(_mm512_maskz_mov_epi32(has_sticky, _mm512_set1_epi32(1)), lsb);
__m512i do_round = _mm512_and_si512(round_bit, sticky_or_lsb);
mant3 = _mm512_add_epi32(mant3, do_round);
// Mantissa overflow (mant3 == 8) -> increment exponent, zero mantissa
__mmask16 mant_ovf = _mm512_cmpeq_epi32_mask(mant3, _mm512_set1_epi32(8));
mant3 = _mm512_mask_mov_epi32(mant3, mant_ovf, _mm512_setzero_si512());
fp8_exp = _mm512_mask_add_epi32(fp8_exp, mant_ovf, fp8_exp, _mm512_set1_epi32(1));
// Clamp exponent: max 15
fp8_exp = _mm512_min_epi32(fp8_exp, _mm512_set1_epi32(15));
// Result: (fp8_exp << 3) | mant3
__m512i result = _mm512_or_si512(_mm512_slli_epi32(fp8_exp, 3), mant3);
// Flush to zero: if f32_exp < 121 (would be subnormal in fp8), output 0
__mmask16 is_subnormal_or_zero = _mm512_cmplt_epi32_mask(f32_exp, _mm512_set1_epi32(121));
result = _mm512_mask_mov_epi32(result, is_subnormal_or_zero, _mm512_setzero_si512());
// Add sign
result = _mm512_or_si512(result, signs);
// Pack 16 x i32 -> 16 x i8
return _mm512_cvtepi32_epi8(result);
}
// Process one tile (64 bf16 values): find amax, compute scale, quantize to fp8.
inline uint8_t quantize_tile_avx512(const at::BFloat16* __restrict__ src, uint8_t* __restrict__ dst) {
// Load 64 bf16 -> 4 x 16 floats
__m256i bf16_0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src));
__m256i bf16_1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 16));
__m256i bf16_2 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 32));
__m256i bf16_3 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 48));
__m512 f0 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_0), 16));
__m512 f1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_1), 16));
__m512 f2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_2), 16));
__m512 f3 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_3), 16));
// Absolute values
const __m512i abs_mask_i = _mm512_set1_epi32(0x7FFFFFFF);
__m512 abs0 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f0), abs_mask_i));
__m512 abs1 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f1), abs_mask_i));
__m512 abs2 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f2), abs_mask_i));
__m512 abs3 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f3), abs_mask_i));
// Horizontal max
__m512 max01 = _mm512_max_ps(abs0, abs1);
__m512 max23 = _mm512_max_ps(abs2, abs3);
__m512 max0123 = _mm512_max_ps(max01, max23);
float amax = _mm512_reduce_max_ps(max0123);
// Scale computation
float scale = std::max(amax / kFp8Max, kQuantEps);
float ceil_log2 = std::ceil(std::log2(scale));
float scale_pow2 = std::exp2(ceil_log2);
float scale_inv = 1.0f / scale_pow2;
int exponent = static_cast<int>(ceil_log2);
uint8_t scale_uint8 = static_cast<uint8_t>(exponent + 127);
// Scale all values
__m512 vinv = _mm512_set1_ps(scale_inv);
__m512 s0 = _mm512_mul_ps(f0, vinv);
__m512 s1 = _mm512_mul_ps(f1, vinv);
__m512 s2 = _mm512_mul_ps(f2, vinv);
__m512 s3 = _mm512_mul_ps(f3, vinv);
// Clamp
__m512 vmax = _mm512_set1_ps(kFp8Max);
__m512 vmin = _mm512_set1_ps(kFp8Min);
s0 = _mm512_max_ps(_mm512_min_ps(s0, vmax), vmin);
s1 = _mm512_max_ps(_mm512_min_ps(s1, vmax), vmin);
s2 = _mm512_max_ps(_mm512_min_ps(s2, vmax), vmin);
s3 = _mm512_max_ps(_mm512_min_ps(s3, vmax), vmin);
// Vectorized float -> fp8 conversion (16 values at a time)
__m128i fp8_0 = cvt_fp32x16_to_fp8x16(s0);
__m128i fp8_1 = cvt_fp32x16_to_fp8x16(s1);
__m128i fp8_2 = cvt_fp32x16_to_fp8x16(s2);
__m128i fp8_3 = cvt_fp32x16_to_fp8x16(s3);
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), fp8_0);
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), fp8_1);
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), fp8_2);
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), fp8_3);
return scale_uint8;
}
#endif // CPU_CAPABILITY_AVX512
template <typename scalar_t>
static void qnorm_rope_kv_insert_impl(
scalar_t* __restrict__ q_out, // [num_tokens_full, q_head_padded, head_dim]
uint8_t* __restrict__ cache, // [num_blocks, cache_row_stride] bytes
const scalar_t* __restrict__ q, // [num_tokens_full, num_heads_q, head_dim]
const scalar_t* __restrict__ kv, // [>=num_tokens_insert, head_dim]
const int64_t* __restrict__ positions, // [num_tokens_full]
const int64_t* __restrict__ slot_mapping, // [num_tokens_insert]
const float* __restrict__ cos_sin_cache, // [max_pos, rope_dim] (first half cos, second half sin)
int64_t num_tokens_full,
int64_t num_heads_q,
int64_t q_head_padded,
int64_t head_dim,
int64_t nope_dim,
int64_t num_quant_blocks,
int64_t token_data_bytes,
int64_t scale_bytes_per_token,
int64_t num_tokens_insert,
int64_t cache_block_size,
int64_t cache_row_stride,
float eps) {
const int64_t rope_half = kRopeDim / 2;
// Q: per-head weight-free RMSNorm + GPT-J RoPE on the last rope_dim dims.
at::parallel_for(0, num_tokens_full * num_heads_q, 0, [&](int64_t begin, int64_t end) {
std::vector<float> buf(head_dim);
for (int64_t idx = begin; idx < end; ++idx) {
const int64_t t = idx / num_heads_q;
const int64_t h = idx % num_heads_q;
const scalar_t* q_th = q + (t * num_heads_q + h) * head_dim;
float sq_sum = 0.f;
for (int64_t i = 0; i < head_dim; ++i) {
const float v = static_cast<float>(q_th[i]);
buf[i] = v;
sq_sum += v * v;
}
const float inv_rms = 1.0f / std::sqrt(sq_sum / static_cast<float>(head_dim) + eps);
for (int64_t i = 0; i < head_dim; ++i)
buf[i] *= inv_rms;
const int64_t pos = positions[t];
const float* cos = cos_sin_cache + pos * kRopeDim;
const float* sin = cos + rope_half;
apply_gptj_rope_inplace(buf.data() + nope_dim, cos, sin, rope_half);
scalar_t* out_th = q_out + (t * q_head_padded + h) * head_dim;
for (int64_t i = 0; i < head_dim; ++i)
out_th[i] = static_cast<scalar_t>(buf[i]);
}
});
// KV: GPT-J RoPE (no RMSNorm) + bf16 round-trip + UE8M0 FP8 quant + paged insert.
at::parallel_for(0, num_tokens_insert, 0, [&](int64_t begin, int64_t end) {
std::vector<float> buf(head_dim);
std::vector<at::BFloat16> nope_bf16(nope_dim);
for (int64_t t = begin; t < end; ++t) {
const int64_t slot_id = slot_mapping[t];
if (slot_id < 0) continue;
const scalar_t* kv_t = kv + t * head_dim;
for (int64_t i = 0; i < head_dim; ++i)
buf[i] = static_cast<float>(kv_t[i]);
const int64_t pos = positions[t];
const float* cos = cos_sin_cache + pos * kRopeDim;
const float* sin = cos + rope_half;
apply_gptj_rope_inplace(buf.data() + nope_dim, cos, sin, rope_half);
// bf16 round-trip, matching the CUDA kernel's double convert.
for (int64_t i = 0; i < head_dim; ++i)
buf[i] = static_cast<float>(at::BFloat16(buf[i]));
for (int64_t i = 0; i < nope_dim; ++i)
nope_bf16[i] = at::BFloat16(buf[i]);
const int64_t block_idx = slot_id / cache_block_size;
const int64_t pos_in_block = slot_id % cache_block_size;
uint8_t* const block_base = cache + block_idx * cache_row_stride;
uint8_t* const token_fp8_ptr = block_base + pos_in_block * token_data_bytes;
uint8_t* const token_bf16_ptr = token_fp8_ptr + nope_dim;
uint8_t* const token_scale_ptr =
block_base + cache_block_size * token_data_bytes + pos_in_block * scale_bytes_per_token;
// AVX512 tile-quantization (SGLang's `quantize_tile_avx512`), one
// 64-wide tile per iteration.
for (int64_t qb = 0; qb < num_quant_blocks; ++qb) {
const at::BFloat16* tile_src = nope_bf16.data() + qb * kQuantBlock;
uint8_t* tile_dst = token_fp8_ptr + qb * kQuantBlock;
token_scale_ptr[qb] = quantize_tile_avx512(tile_src, tile_dst);
}
token_scale_ptr[num_quant_blocks] = 0;
auto* out_rope = reinterpret_cast<at::BFloat16*>(token_bf16_ptr);
for (int64_t i = 0; i < kRopeDim; ++i)
out_rope[i] = at::BFloat16(buf[nope_dim + i]);
}
});
}
} // anonymous namespace
at::Tensor fused_qnorm_rope_kv_insert_cpu(
at::Tensor& q,
at::Tensor& kv,
at::Tensor& positions,
at::Tensor& swa_kv_cache_2d,
at::Tensor& slot_mapping,
at::Tensor& cos_sin_cache,
int64_t q_head_padded,
double eps,
int64_t cache_block_size) {
TORCH_CHECK(
q.dim() == 3 && (q.scalar_type() == at::kBFloat16 || q.scalar_type() == at::kHalf),
"fused_qnorm_rope_kv_insert_cpu: q must be bf16/fp16 [num_tokens, num_heads_q, head_dim]");
TORCH_CHECK(kv.dim() == 2 && kv.scalar_type() == q.scalar_type(), "fused_qnorm_rope_kv_insert_cpu: kv dtype must match q");
TORCH_CHECK(positions.scalar_type() == at::kLong, "fused_qnorm_rope_kv_insert_cpu: positions must be int64");
TORCH_CHECK(slot_mapping.scalar_type() == at::kLong, "fused_qnorm_rope_kv_insert_cpu: slot_mapping must be int64");
TORCH_CHECK(
swa_kv_cache_2d.dim() == 2 && swa_kv_cache_2d.scalar_type() == at::kByte,
"fused_qnorm_rope_kv_insert_cpu: swa_kv_cache_2d must be a 2D uint8 tensor");
TORCH_CHECK(
q.device().is_cpu() && kv.device().is_cpu() && positions.device().is_cpu() && slot_mapping.device().is_cpu() &&
swa_kv_cache_2d.device().is_cpu() && cos_sin_cache.device().is_cpu(),
"fused_qnorm_rope_kv_insert_cpu: all inputs must be CPU tensors");
const int64_t num_tokens_full = q.size(0);
const int64_t num_heads_q = q.size(1);
const int64_t head_dim = q.size(2);
const int64_t nope_dim = head_dim - kRopeDim;
TORCH_CHECK(nope_dim > 0 && nope_dim % kQuantBlock == 0, "fused_qnorm_rope_kv_insert_cpu: unexpected head_dim");
const int64_t num_quant_blocks = nope_dim / kQuantBlock;
const int64_t scale_bytes_per_token = num_quant_blocks + 1;
const int64_t token_data_bytes = nope_dim + kRopeDim * 2;
TORCH_CHECK(q_head_padded >= num_heads_q, "fused_qnorm_rope_kv_insert_cpu: q_head_padded must be >= num_heads_q");
TORCH_CHECK(kv.size(1) == head_dim, "fused_qnorm_rope_kv_insert_cpu: kv head_dim must match q");
TORCH_CHECK(positions.numel() == num_tokens_full, "fused_qnorm_rope_kv_insert_cpu: positions must have num_tokens_full entries");
const int64_t num_tokens_insert = slot_mapping.size(0);
TORCH_CHECK(kv.size(0) >= num_tokens_insert, "fused_qnorm_rope_kv_insert_cpu: kv must have >= num_tokens_insert rows");
TORCH_CHECK(cache_block_size > 0, "fused_qnorm_rope_kv_insert_cpu: cache_block_size must be positive");
TORCH_CHECK(
cos_sin_cache.scalar_type() == at::kFloat && cos_sin_cache.is_contiguous(),
"fused_qnorm_rope_kv_insert_cpu: cos_sin_cache must be float32 and "
"contiguous (convert once in Python)");
TORCH_CHECK(
q.is_contiguous() && kv.is_contiguous() && positions.is_contiguous() && slot_mapping.is_contiguous(),
"fused_qnorm_rope_kv_insert_cpu: q/kv/positions/slot_mapping must be contiguous");
// swa_kv_cache_2d is mutated in place; index via its real row stride
// rather than assuming it equals cache_block_size * (token_data_bytes +
// scale_bytes_per_token) -- a paged/pooled buffer's row stride is not
// guaranteed to match the logical per-block byte count.
const int64_t cache_row_stride = swa_kv_cache_2d.stride(0);
auto q_out = at::zeros({num_tokens_full, q_head_padded, head_dim}, q.options());
AT_DISPATCH_REDUCED_FLOATING_TYPES(q.scalar_type(), "fused_qnorm_rope_kv_insert_cpu", [&] {
qnorm_rope_kv_insert_impl<scalar_t>(
q_out.data_ptr<scalar_t>(),
swa_kv_cache_2d.data_ptr<uint8_t>(),
q.data_ptr<scalar_t>(),
kv.data_ptr<scalar_t>(),
positions.data_ptr<int64_t>(),
slot_mapping.data_ptr<int64_t>(),
cos_sin_cache.data_ptr<float>(),
num_tokens_full,
num_heads_q,
q_head_padded,
head_dim,
nope_dim,
num_quant_blocks,
token_data_bytes,
scale_bytes_per_token,
num_tokens_insert,
cache_block_size,
cache_row_stride,
static_cast<float>(eps));
});
return q_out;
}
// vLLM-native: inverse GPT-J RoPE for `_o_proj`'s output de-rotation (undoes
// the query's own rotation on the attention output before the low-rank
// wo_a/wo_b projection -- see the math writeup in cpu_sparse.py's `_o_proj`).
// Upstream needs this same step (`fused_rope_inplace(..., inverse=True)` in
// `deepseek_v4.py`) but only has a CUDA-JIT implementation, no CPU kernel --
// this fills that gap rather than porting one. Replaces the eager reference
// in `DeepseekV4CPUAttention._o_proj` (cpu_sparse.py). Uses the same
// half-half `[cos...|sin...]` cache layout as `apply_gptj_rope_inplace`
// above (not upstream's interleaved `[c0,s0,c1,s1,...]` layout).
//
// AVX512 path: NoPE dtype-convert copy uses `vec.h`'s `load_float_vec`
// (16-wide bf16/fp16 -> fp32). The interleaved (even,odd) rotation widens
// 32 raw elements to fp32 via `load_float_vec2`, round-trips them through
// an aligned scratch buffer, then gathers/scatters the even/odd lanes --
// same technique as `indexer.cpp`'s Q-side RoPE, since `cos`/`sin` here
// are two contiguous halves rather than per-pair-interleaved.
at::Tensor inverse_gptj_rope_o_proj_cpu(
at::Tensor& o, // [num_tokens, num_heads, head_dim]
at::Tensor& positions, // [num_tokens]
at::Tensor& cos_sin_cache, // [max_pos, rope_dim] (first half cos, second half sin)
int64_t rope_dim) {
TORCH_CHECK(o.dim() == 3, "inverse_gptj_rope_o_proj_cpu: o must be 3D [num_tokens, num_heads, head_dim]");
TORCH_CHECK(positions.scalar_type() == at::kLong, "inverse_gptj_rope_o_proj_cpu: positions must be int64");
TORCH_CHECK(
o.device().is_cpu() && positions.device().is_cpu() && cos_sin_cache.device().is_cpu(),
"inverse_gptj_rope_o_proj_cpu: all inputs must be CPU tensors");
const int64_t num_tokens = o.size(0);
const int64_t num_heads = o.size(1);
const int64_t head_dim = o.size(2);
const int64_t nope_dim = head_dim - rope_dim;
const int64_t half_dim = rope_dim / 2;
TORCH_CHECK(nope_dim >= 0, "inverse_gptj_rope_o_proj_cpu: rope_dim must not exceed head_dim");
// Only ever called with head_dim=512, rope_dim=64 (DeepseekV4CPUAttention's
// _o_proj); the AVX512 loops below process 16 elements/pairs per iteration
// with no scalar remainder, so require both to divide evenly.
TORCH_CHECK(nope_dim % 16 == 0 && half_dim % 16 == 0,
"inverse_gptj_rope_o_proj_cpu: head_dim-rope_dim and rope_dim/2 must be multiples of 16");
TORCH_CHECK(
cos_sin_cache.scalar_type() == at::kFloat && cos_sin_cache.is_contiguous(),
"inverse_gptj_rope_o_proj_cpu: cos_sin_cache must be float32 and "
"contiguous (convert once in Python)");
TORCH_CHECK(o.is_contiguous() && positions.is_contiguous(),
"inverse_gptj_rope_o_proj_cpu: o/positions must be contiguous");
auto out = at::empty({num_tokens, num_heads, head_dim}, o.options().dtype(at::kFloat));
AT_DISPATCH_REDUCED_FLOATING_TYPES(o.scalar_type(), "inverse_gptj_rope_o_proj_cpu", [&] {
const scalar_t* o_ptr = o.data_ptr<scalar_t>();
float* out_ptr = out.data_ptr<float>();
const int64_t* pos_ptr = positions.data_ptr<int64_t>();
const float* cos_sin_ptr = cos_sin_cache.data_ptr<float>();
at::parallel_for(0, num_tokens * num_heads, 0, [&](int64_t begin, int64_t end) {
for (int64_t idx = begin; idx < end; ++idx) {
const int64_t t = idx / num_heads;
const int64_t h = idx % num_heads;
const scalar_t* src = o_ptr + (t * num_heads + h) * head_dim;
float* dst = out_ptr + (t * num_heads + h) * head_dim;
// head_dim=512, rope_dim=64 in practice (checked above), so
// nope_dim/half_dim are always multiples of 16 -- no scalar
// remainder needed.
for (int64_t i = 0; i < nope_dim; i += 16) {
load_float_vec<scalar_t>(src + i).store(dst + i);
}
const int64_t pos = pos_ptr[t];
const float* cos = cos_sin_ptr + pos * rope_dim;
const float* sin = cos + half_dim;
{
// Permute-based (no gather/scatter), adapted from SGLang's
// `rope.cpp` (`apply_rotary_emb_row`) the same way as
// compressor.cpp's `apply_gptj_rope_inplace`: `_mm512_permute_ps`
// swaps each pair's (even, odd) lanes in one shuffle, and cos/sin
// -- loaded as 8 contiguous values -- are duplicated pairwise via
// `_mm512_permutexvar_ps` to align with that layout, since our
// cos/sin are two separate flat arrays rather than SGLang's
// pre-interleaved freqs. Two batches of 8 pairs per iteration
// (matching `load_float_vec2`'s 16+16-lane split).
const __m512i dup_idx = _mm512_set_epi32(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0);
// Inverse GPT-J rotation negates the sin broadcast on the odd
// (imag) lane of each pair; the even (real) lane keeps sin's sign.
const __m512 sign_mask = _mm512_castsi512_ps(_mm512_set_epi32(
(int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0,
(int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0));
for (int64_t j = 0; j < half_dim; j += 16) {
const scalar_t* rsrc = src + nope_dim + 2 * j;
at::vec::Vectorized<float> x0, x1;
std::tie(x0, x1) = load_float_vec2(rsrc);
float* rdst = dst + nope_dim + 2 * j;
__m512 xv0 = x0;
__m512 cos_b0 = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(cos + j)));
__m512 sin_b0 = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(sin + j)));
__m512 out0 = _mm512_fmadd_ps(
xv0, cos_b0, _mm512_mul_ps(_mm512_permute_ps(xv0, 0xB1), _mm512_xor_ps(sin_b0, sign_mask)));
_mm512_storeu_ps(rdst, out0);
__m512 xv1 = x1;
__m512 cos_b1 = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(cos + j + 8)));
__m512 sin_b1 = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(sin + j + 8)));
__m512 out1 = _mm512_fmadd_ps(
xv1, cos_b1, _mm512_mul_ps(_mm512_permute_ps(xv1, 0xB1), _mm512_xor_ps(sin_b1, sign_mask)));
_mm512_storeu_ps(rdst + 16, out1);
}
}
}
});
});
return out;
}
+834
View File
@@ -0,0 +1,834 @@
// Adapted from
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu:
// `biased_topk_kernel_impl`/`biased_topk_cpu` (flat, non-grouped DeepSeek-V4
// biased top-k) and `hash_topk_kernel_impl`/`hash_topk_cpu` (hash-routed
// layers, expert IDs from a precomputed `tid2eid` lookup table), plus their
// `sigmoid`/`softmax`/`apply_bias`/`sqrtsoftplus` scoring helpers. See
// `biased_topk_cpu`/`hash_topk_cpu` below for the one deliberate deviation
// from the upstream kernel (routed_scaling_factor handling when no shared
// expert is fused into routing, vLLM's CPU DeepSeek-V4 calling convention).
//
// clang-format off
#include <algorithm>
#include <array>
#include <cmath>
#include <limits>
#include <type_traits>
#include <utility>
#include "common.h"
#include "vec.h"
namespace {
template <typename scalar_t, int SIZE, std::enable_if_t<!std::is_same_v<scalar_t, float>, int> = 0>
inline void sigmoid(float* __restrict__ out, const scalar_t* __restrict__ input) {
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
const fVec one = fVec(1.f);
constexpr int kVecSize = bVec::size();
for (int d = 0; d < SIZE; d += kVecSize) {
bVec x_bvec = bVec::loadu(input + d);
fVec x_fvec0, x_fvec1;
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
x_fvec0 = one / (one + x_fvec0.neg().exp_u20());
x_fvec1 = one / (one + x_fvec1.neg().exp_u20());
x_fvec0.store(out + d);
x_fvec1.store(out + d + fVec::size());
}
}
template <typename scalar_t, int SIZE, std::enable_if_t<std::is_same_v<scalar_t, float>, int> = 0>
inline void sigmoid(float* __restrict__ out, const float* __restrict__ input) {
using fVec = at::vec::Vectorized<float>;
const fVec one = fVec(1.f);
constexpr int kVecSize = fVec::size();
for (int d = 0; d < SIZE; d += kVecSize) {
fVec in_fvec = fVec::loadu(input + d);
in_fvec = one / (one + in_fvec.neg().exp_u20());
in_fvec.store(out + d);
}
}
template <typename scalar_t, int SIZE, std::enable_if_t<!std::is_same_v<scalar_t, float>, int> = 0>
inline void softmax(float* __restrict__ out, const scalar_t* __restrict__ input) {
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
constexpr int kVecSize = bVec::size();
// step 1: get max
fVec max_fvec = fVec(-std::numeric_limits<float>::infinity());
for (int d = 0; d < SIZE; d += kVecSize) {
bVec x_bvec = bVec::loadu(input + d);
fVec x_fvec0, x_fvec1;
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
max_fvec = at::vec::maximum(max_fvec, x_fvec0);
max_fvec = at::vec::maximum(max_fvec, x_fvec1);
x_fvec0.store(out + d);
x_fvec1.store(out + d + fVec::size());
}
float max_val = vec_reduce_max(max_fvec);
max_fvec = fVec(max_val);
// step 2: sum of (x - max).exp()
fVec sum_fvec = fVec(float(0));
for (int d = 0; d < SIZE; d += fVec::size()) {
fVec x_fvec = (fVec::loadu(out + d) - max_fvec).exp_u20();
sum_fvec += x_fvec;
x_fvec.store(out + d);
}
float sum_val = vec_reduce_sum(sum_fvec);
// step 3: x * (1 / sum)
sum_fvec = fVec(1.f / sum_val);
for (int d = 0; d < SIZE; d += fVec::size()) {
fVec out_fvec = fVec::loadu(out + d) * sum_fvec;
out_fvec.store(out + d);
}
}
template <typename scalar_t, int SIZE, std::enable_if_t<std::is_same_v<scalar_t, float>, int> = 0>
inline void softmax(float* __restrict__ out, const float* __restrict__ input) {
using fVec = at::vec::Vectorized<float>;
constexpr int kVecSize = fVec::size();
// step 1: get max
fVec max_fvec = fVec(-std::numeric_limits<float>::infinity());
for (int d = 0; d < SIZE; d += kVecSize) {
fVec x_fvec = fVec::loadu(input + d);
max_fvec = at::vec::maximum(max_fvec, x_fvec);
x_fvec.store(out + d);
}
float max_val = vec_reduce_max(max_fvec);
max_fvec = fVec(max_val);
// step 2: sum of (x - max).exp()
fVec sum_fvec = fVec(float(0));
for (int d = 0; d < SIZE; d += kVecSize) {
fVec x_fvec = (fVec::loadu(out + d) - max_fvec).exp_u20();
sum_fvec += x_fvec;
x_fvec.store(out + d);
}
float sum_val = vec_reduce_sum(sum_fvec);
// step 3: x * (1 / sum)
sum_fvec = fVec(1.f / sum_val);
for (int d = 0; d < SIZE; d += kVecSize) {
fVec out_fvec = fVec::loadu(out + d) * sum_fvec;
out_fvec.store(out + d);
}
}
template <typename param_t, int SIZE>
inline void
apply_bias(float* __restrict__ scores2, const float* __restrict__ scores, const param_t* __restrict__ bias) {
using fVec = at::vec::Vectorized<float>;
auto vec_size = fVec::size() * 2;
int d = 0;
for (; d <= SIZE - vec_size; d += vec_size) {
fVec bias0, bias1, x0, x1;
std::tie(bias0, bias1) = load_float_vec2(bias + d);
std::tie(x0, x1) = load_float_vec2(scores + d);
x0 = x0 + bias0;
x1 = x1 + bias1;
x0.store(scores2 + d);
x1.store(scores2 + d + fVec::size());
}
for (; d < SIZE; d++) {
scores2[d] = scores[d] + (float)bias[d];
}
}
// sqrtsoftplus: sqrt(softplus(x)) = sqrt(log(1 + exp(x)))
// For numerical stability: when x > threshold, softplus(x) ≈ x
// When x < -threshold, softplus(x) ≈ exp(x), so sqrt(softplus(x)) ≈ sqrt(exp(x)) = exp(x/2)
template <typename scalar_t, int SIZE, std::enable_if_t<!std::is_same_v<scalar_t, float>, int> = 0>
inline void sqrtsoftplus(float* __restrict__ out, const scalar_t* __restrict__ input) {
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
const fVec one = fVec(1.f);
const fVec half = fVec(0.5f);
const fVec threshold = fVec(20.f);
const fVec neg_threshold = fVec(-15.f); // below this, 1+exp(x) loses precision in float32
constexpr int kVecSize = bVec::size();
for (int d = 0; d < SIZE; d += kVecSize) {
bVec x_bvec = bVec::loadu(input + d);
fVec x_fvec0, x_fvec1;
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
// default: softplus(x) = log(1 + exp(x))
fVec sp0 = (one + x_fvec0.exp_u20()).log();
fVec sp1 = (one + x_fvec1.exp_u20()).log();
// x > 20: softplus(x) ≈ x
sp0 = fVec::blendv(sp0, x_fvec0, x_fvec0 > threshold);
sp1 = fVec::blendv(sp1, x_fvec1, x_fvec1 > threshold);
// x < -15: softplus(x) ≈ exp(x), sqrt(exp(x)) = exp(x/2)
fVec exp_half0 = (x_fvec0 * half).exp_u20();
fVec exp_half1 = (x_fvec1 * half).exp_u20();
sp0 = fVec::blendv(sp0.sqrt(), exp_half0, x_fvec0 < neg_threshold);
sp1 = fVec::blendv(sp1.sqrt(), exp_half1, x_fvec1 < neg_threshold);
sp0.store(out + d);
sp1.store(out + d + fVec::size());
}
}
template <typename scalar_t, int SIZE, std::enable_if_t<std::is_same_v<scalar_t, float>, int> = 0>
inline void sqrtsoftplus(float* __restrict__ out, const float* __restrict__ input) {
using fVec = at::vec::Vectorized<float>;
const fVec one = fVec(1.f);
const fVec half = fVec(0.5f);
const fVec threshold = fVec(20.f);
const fVec neg_threshold = fVec(-15.f);
constexpr int kVecSize = fVec::size();
for (int d = 0; d < SIZE; d += kVecSize) {
fVec x = fVec::loadu(input + d);
fVec sp = (one + x.exp_u20()).log();
sp = fVec::blendv(sp, x, x > threshold);
fVec exp_half = (x * half).exp_u20();
sp = fVec::blendv(sp.sqrt(), exp_half, x < neg_threshold);
sp.store(out + d);
}
}
// biased_topk: flat (non-grouped) biased topk for DeepSeek V4
// scoring_func: 0 = sigmoid, 1 = sqrtsoftplus
template <typename scalar_t, typename param_t, int NUM_EXPERTS, int TOPK>
void biased_topk_kernel_impl(
float* __restrict__ topk_weights,
int32_t* __restrict__ topk_ids,
const scalar_t* __restrict__ gating_output,
const param_t* __restrict__ bias,
int64_t num_tokens,
int64_t topk,
bool renormalize,
int scoring_func,
int64_t num_fused_shared_experts,
float routed_scaling_factor,
bool apply_routed_scaling_factor_on_output) {
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
alignas(64) float scores[NUM_EXPERTS];
alignas(64) float scores_biased[NUM_EXPERTS];
using elem_t = std::pair<float, int32_t>;
std::array<elem_t, NUM_EXPERTS> queue;
// simple RNG for fused shared expert random ID
uint64_t rng_state = begin * 6364136223846793005ULL + 1442695040888963407ULL;
for (int64_t i = begin; i < end; ++i) {
// compute scores
if (scoring_func == 0) {
sigmoid<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
} else {
sqrtsoftplus<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
}
// add bias for selection
apply_bias<param_t, NUM_EXPERTS>(scores_biased, scores, bias);
// build queue and partial sort to find top-k
for (int64_t e = 0; e < NUM_EXPERTS; ++e) {
queue[e] = {scores_biased[e], static_cast<int32_t>(e)};
}
std::partial_sort(
queue.begin(), queue.begin() + topk, queue.end(), [](const elem_t& x, const elem_t& y) -> bool {
return x.first > y.first;
});
// gather original scores (without bias) as weights
for (int64_t j = 0; j < topk; ++j) {
int32_t idx = queue[j].second;
topk_ids[i * topk + j] = idx;
topk_weights[i * topk + j] = scores[idx];
}
// handle fused shared experts
if (num_fused_shared_experts > 0) {
// replace last slot with random shared expert ID
rng_state = rng_state * 6364136223846793005ULL + 1442695040888963407ULL;
int32_t shared_id =
NUM_EXPERTS + static_cast<int32_t>((rng_state >> 33) % static_cast<uint64_t>(num_fused_shared_experts));
topk_ids[i * topk + topk - 1] = shared_id;
// shared expert weight = sum of routed weights / scaling_factor
if (routed_scaling_factor != 0.0f) {
float routed_sum = 0.f;
for (int64_t j = 0; j < topk - 1; ++j) {
routed_sum += topk_weights[i * topk + j];
}
topk_weights[i * topk + topk - 1] = routed_sum / routed_scaling_factor;
}
}
// renormalize
if (renormalize) {
float sum = 0.f;
int64_t norm_end = (num_fused_shared_experts == 0) ? topk : topk - 1;
for (int64_t j = 0; j < norm_end; ++j) {
sum += topk_weights[i * topk + j];
}
float scale = 1.f / sum;
if (apply_routed_scaling_factor_on_output) {
scale *= routed_scaling_factor;
}
for (int64_t j = 0; j < norm_end; ++j) {
topk_weights[i * topk + j] *= scale;
}
// also scale the shared expert weight if present
if (num_fused_shared_experts > 0) {
topk_weights[i * topk + topk - 1] *= scale;
}
}
}
});
}
// hash_topk: expert IDs come from a precomputed lookup table tid2eid[input_ids]
// scoring_func: 0 = softmax, 1 = sigmoid, 2 = sqrtsoftplus
template <typename scalar_t, int NUM_EXPERTS, int TOPK>
void hash_topk_kernel_impl(
float* __restrict__ topk_weights,
int32_t* __restrict__ topk_ids,
const scalar_t* __restrict__ gating_output,
const int32_t* __restrict__ tid2eid, // [num_tokens, routed_topk]
int64_t num_tokens,
int scoring_func,
int64_t num_fused_shared_experts,
int64_t num_experts,
float routed_scaling_factor,
int64_t topk) {
const int64_t routed_topk = topk - num_fused_shared_experts;
const bool need_renormalize = (scoring_func != 0); // renormalize for non-softmax
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
alignas(64) float scores[NUM_EXPERTS];
// simple RNG for fused shared expert random ID
uint64_t rng_state = begin * 6364136223846793005ULL + 1442695040888963407ULL;
for (int64_t i = begin; i < end; ++i) {
// compute scores over all experts
if (scoring_func == 0) {
softmax<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
} else if (scoring_func == 1) {
sigmoid<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
} else {
sqrtsoftplus<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
}
// gather expert IDs from lookup table
const int32_t* eid_row = tid2eid + i * routed_topk;
for (int64_t j = 0; j < routed_topk; ++j) {
int32_t eid = eid_row[j];
topk_ids[i * topk + j] = eid;
topk_weights[i * topk + j] = scores[eid];
}
// renormalize routed weights (for non-softmax scoring). Unlike the
// upstream kernel, always fold routed_scaling_factor in here rather
// than only applying it to the fused-shared-expert slot below: vLLM's
// DeepSeek-V4 CPU hash-routed layers never fuse a shared expert into
// this routing table (num_fused_shared_experts is always 0 on this
// call path), so this is the only place the scaling factor can be
// applied. Matches the eager reference (`sqrtsoftplus_bias_topk` in
// cpu_moe.py) and the CUDA kernel (`dsv4HashTopkSoftplusSqrt`).
if (need_renormalize) {
float sum = 0.f;
for (int64_t j = 0; j < routed_topk; ++j) {
sum += topk_weights[i * topk + j];
}
if (sum > 0.f) {
float scale = 1.f / sum;
if (num_fused_shared_experts == 0) {
scale *= routed_scaling_factor;
}
for (int64_t j = 0; j < routed_topk; ++j) {
topk_weights[i * topk + j] *= scale;
}
}
}
// handle fused shared expert
if (num_fused_shared_experts > 0) {
// random shared expert ID in [num_experts, num_experts + num_fused_shared_experts)
rng_state = rng_state * 6364136223846793005ULL + 1442695040888963407ULL;
int32_t shared_id =
num_experts + static_cast<int32_t>((rng_state >> 33) % static_cast<uint64_t>(num_fused_shared_experts));
topk_ids[i * topk + topk - 1] = shared_id;
// shared expert weight = sum of routed weights / scaling_factor
float routed_sum = 0.f;
for (int64_t j = 0; j < routed_topk; ++j) {
routed_sum += topk_weights[i * topk + j];
}
topk_weights[i * topk + topk - 1] = routed_sum / routed_scaling_factor;
}
}
});
}
#define LAUNCH_HASH_TOPK_KERNEL(NE, NTOPK) \
hash_topk_kernel_impl<scalar_t, NE, NTOPK>( \
topk_weights.data_ptr<float>(), \
topk_ids.data_ptr<int32_t>(), \
gating_output.data_ptr<scalar_t>(), \
tid2eid_flat.data_ptr<int32_t>(), \
num_tokens, \
scoring_func_id, \
num_fused_shared_experts, \
num_experts, \
routed_scaling_factor_value, \
topk);
#define LAUNCH_BIASED_TOPK_KERNEL(NE, NTOPK) \
biased_topk_kernel_impl<scalar_t, param_t, NE, NTOPK>( \
topk_weights.data_ptr<float>(), \
topk_ids.data_ptr<int32_t>(), \
gating_output.data_ptr<scalar_t>(), \
correction_bias.data_ptr<param_t>(), \
num_tokens, \
topk, \
renormalize, \
scoring_func_id, \
num_fused_shared_experts, \
routed_scaling_factor_value, \
apply_routed_scaling_factor_on_output);
} // namespace
// biased topk for DeepSeek V4 (flat, non-grouped)
std::tuple<at::Tensor, at::Tensor> biased_topk_cpu(
at::Tensor& hidden_states,
at::Tensor& gating_output,
at::Tensor& correction_bias,
int64_t topk,
bool renormalize,
std::string scoring_func,
int64_t num_fused_shared_experts,
std::optional<double> routed_scaling_factor,
bool apply_routed_scaling_factor_on_output) {
CHECK_INPUT(gating_output);
CHECK_INPUT(correction_bias);
const auto st = gating_output.scalar_type();
int64_t num_tokens = hidden_states.size(0);
int64_t num_experts = gating_output.size(1);
TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch");
TORCH_CHECK(correction_bias.numel() == num_experts, "Bias shape mismatch");
int scoring_func_id = 0;
if (scoring_func == "sigmoid") {
scoring_func_id = 0;
} else if (scoring_func == "sqrtsoftplus") {
scoring_func_id = 1;
} else {
TORCH_CHECK(false, "Unsupported scoring_func: ", scoring_func);
}
float routed_scaling_factor_value = routed_scaling_factor.has_value() ? routed_scaling_factor.value() : 0.0f;
at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat));
at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt));
// The actual routed topk (excluding fused shared experts)
int64_t routed_topk = topk - num_fused_shared_experts;
CPU_DISPATCH_FLOATING_TYPES_EXT(st, correction_bias.scalar_type(), "biased_topk_kernel", [&] {
// dispatch on num_experts and routed_topk. For DeepSeek V4: 256 experts,
// routed_topk=6 (num_fused_shared_experts is always 0 on this CPU call
// path -- see cpu_moe.py). Anything outside this table isn't a real
// model shape, so fail loudly instead of silently degrading to a slow
// generic path.
switch (num_experts) {
case 64:
switch (routed_topk) {
case 6:
LAUNCH_BIASED_TOPK_KERNEL(64, 6);
break;
case 8:
LAUNCH_BIASED_TOPK_KERNEL(64, 8);
break;
default:
TORCH_CHECK(false, "biased_topk_cpu: unsupported routed_topk ", routed_topk,
" for num_experts=64");
}
break;
case 128:
switch (routed_topk) {
case 6:
LAUNCH_BIASED_TOPK_KERNEL(128, 6);
break;
case 8:
LAUNCH_BIASED_TOPK_KERNEL(128, 8);
break;
default:
TORCH_CHECK(false, "biased_topk_cpu: unsupported routed_topk ", routed_topk,
" for num_experts=128");
}
break;
case 256:
switch (routed_topk) {
case 6:
LAUNCH_BIASED_TOPK_KERNEL(256, 6);
break;
case 8:
LAUNCH_BIASED_TOPK_KERNEL(256, 8);
break;
case 9:
LAUNCH_BIASED_TOPK_KERNEL(256, 9);
break;
default:
TORCH_CHECK(false, "biased_topk_cpu: unsupported routed_topk ", routed_topk,
" for num_experts=256");
}
break;
case 384:
switch (routed_topk) {
case 6:
LAUNCH_BIASED_TOPK_KERNEL(384, 6);
break;
case 8:
LAUNCH_BIASED_TOPK_KERNEL(384, 8);
break;
default:
TORCH_CHECK(false, "biased_topk_cpu: unsupported routed_topk ", routed_topk,
" for num_experts=384");
}
break;
default:
TORCH_CHECK(false, "biased_topk_cpu: unsupported num_experts ", num_experts);
}
});
return std::make_tuple(topk_weights, topk_ids);
}
// hash topk for DeepSeek V4 (expert IDs from precomputed lookup table)
std::tuple<at::Tensor, at::Tensor> hash_topk_cpu(
at::Tensor& gating_output,
at::Tensor& tid2eid,
int64_t topk,
std::string scoring_func,
int64_t num_fused_shared_experts,
int64_t num_experts,
double routed_scaling_factor) {
CHECK_INPUT(gating_output);
CHECK_INPUT(tid2eid);
const auto st = gating_output.scalar_type();
int64_t num_tokens = gating_output.size(0);
int64_t num_experts_gating = gating_output.size(1);
int64_t routed_topk = topk - num_fused_shared_experts;
TORCH_CHECK(tid2eid.size(0) == num_tokens, "tid2eid row count must match num_tokens");
TORCH_CHECK(tid2eid.size(1) == routed_topk, "tid2eid column count must match routed_topk");
TORCH_CHECK(tid2eid.scalar_type() == at::kInt, "tid2eid must be int32");
TORCH_CHECK(num_experts_gating == num_experts, "num_experts mismatch");
int scoring_func_id = 0;
if (scoring_func == "softmax") {
scoring_func_id = 0;
} else if (scoring_func == "sigmoid") {
scoring_func_id = 1;
} else if (scoring_func == "sqrtsoftplus") {
scoring_func_id = 2;
} else {
TORCH_CHECK(false, "Unsupported scoring_func: ", scoring_func);
}
float routed_scaling_factor_value = static_cast<float>(routed_scaling_factor);
at::Tensor topk_weights = at::empty({num_tokens, topk}, gating_output.options().dtype(at::kFloat));
at::Tensor topk_ids = at::empty({num_tokens, topk}, gating_output.options().dtype(at::kInt));
// tid2eid is [num_tokens, routed_topk], already indexed by input_ids in
// Python; CHECK_INPUT above already guarantees it is contiguous.
const at::Tensor& tid2eid_flat = tid2eid;
// Dispatch for bf16, fp16, and float32
// Note: cannot use AT_DISPATCH_FLOATING_TYPES_AND2 since it includes double which lacks convert_to_float
auto dispatch_fn = [&]<typename scalar_t>() {
// For DeepSeek V4: 256 experts, routed_topk in {6,7,8} depending on
// layer (num_fused_shared_experts is always 0 on this CPU call path --
// see cpu_moe.py). Anything outside this table isn't a real model
// shape, so fail loudly instead of silently degrading to a slow
// generic path.
switch (num_experts) {
case 64:
switch (routed_topk) {
case 6:
LAUNCH_HASH_TOPK_KERNEL(64, 6);
break;
case 7:
LAUNCH_HASH_TOPK_KERNEL(64, 7);
break;
case 8:
LAUNCH_HASH_TOPK_KERNEL(64, 8);
break;
default:
TORCH_CHECK(false, "hash_topk_cpu: unsupported routed_topk ", routed_topk,
" for num_experts=64");
}
break;
case 128:
switch (routed_topk) {
case 6:
LAUNCH_HASH_TOPK_KERNEL(128, 6);
break;
case 7:
LAUNCH_HASH_TOPK_KERNEL(128, 7);
break;
case 8:
LAUNCH_HASH_TOPK_KERNEL(128, 8);
break;
default:
TORCH_CHECK(false, "hash_topk_cpu: unsupported routed_topk ", routed_topk,
" for num_experts=128");
}
break;
case 256:
switch (routed_topk) {
case 6:
LAUNCH_HASH_TOPK_KERNEL(256, 6);
break;
case 7:
LAUNCH_HASH_TOPK_KERNEL(256, 7);
break;
case 8:
LAUNCH_HASH_TOPK_KERNEL(256, 8);
break;
default:
TORCH_CHECK(false, "hash_topk_cpu: unsupported routed_topk ", routed_topk,
" for num_experts=256");
}
break;
case 384:
switch (routed_topk) {
case 6:
LAUNCH_HASH_TOPK_KERNEL(384, 6);
break;
case 7:
LAUNCH_HASH_TOPK_KERNEL(384, 7);
break;
case 8:
LAUNCH_HASH_TOPK_KERNEL(384, 8);
break;
default:
TORCH_CHECK(false, "hash_topk_cpu: unsupported routed_topk ", routed_topk,
" for num_experts=384");
}
break;
default:
TORCH_CHECK(false, "hash_topk_cpu: unsupported num_experts ", num_experts);
}
};
if (st == at::ScalarType::BFloat16) {
dispatch_fn.template operator()<at::BFloat16>();
} else if (st == at::ScalarType::Half) {
dispatch_fn.template operator()<at::Half>();
} else if (st == at::ScalarType::Float) {
dispatch_fn.template operator()<float>();
} else {
TORCH_CHECK(false, "Unsupported dtype for hash_topk_cpu: ", st);
}
return std::make_tuple(topk_weights, topk_ids);
}
// Adapted from
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu:
// `topk_transform_512_cpu`/`topk_transform_512_cpu_kernel_impl`. Near-
// verbatim port for the sparse indexer's DECODE path: hardcodes top-512
// (the only `index_topk` DeepSeek-V4-Flash uses) and streams a min-heap over
// the tail of the row instead of sorting the whole row.
//
// Differences from upstream:
// - `out_page_indices` is still computed (required by the kernel contract)
// but vLLM's caller only consumes `out_raw_indices` (local/compressed-
// context positions -- `DeepseekV4CPUAttention.forward_mqa` resolves those
// to physical slots itself via `map_local_to_global_slots_cpu`).
// - `page_size` was already a runtime argument upstream, unlike
// `fp8_paged_mqa_logits_cpu` (paged_mqa_logits.cpp), which needed one.
namespace {
constexpr int64_t kC4Topk = 512;
template <typename T>
inline int64_t load_index_value(const T* __restrict__ ptr, int64_t idx) {
return static_cast<int64_t>(ptr[idx]);
}
struct TopKTransformElem {
float score;
int32_t index;
};
struct TopKTransformMinHeapCmp {
bool operator()(const TopKTransformElem& lhs, const TopKTransformElem& rhs) const {
if (lhs.score == rhs.score) {
return lhs.index > rhs.index;
}
return lhs.score > rhs.score;
}
};
template <typename seq_t, typename page_t>
void topk_transform_512_cpu_kernel_impl(
const float* __restrict__ scores,
const seq_t* __restrict__ seq_lens,
const page_t* __restrict__ page_tables,
int32_t* __restrict__ out_page_indices,
int32_t* __restrict__ out_raw_indices,
int64_t batch_size,
int64_t max_seq_len,
int64_t page_table_stride,
int64_t out_stride,
int64_t page_size) {
TORCH_CHECK(page_size > 0, "page_size must be positive");
TORCH_CHECK((page_size & (page_size - 1)) == 0, "page_size must be a power of 2");
const int page_bits = page_size > 1 ? static_cast<int>(std::log2(static_cast<double>(page_size))) : 0;
const int64_t page_mask = page_size - 1;
at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) {
std::array<TopKTransformElem, kC4Topk> heap;
for (int64_t b = begin; b < end; ++b) {
const float* __restrict__ scores_row = scores + b * max_seq_len;
const page_t* __restrict__ page_table_row = page_tables + b * page_table_stride;
int32_t* __restrict__ out_page_row = out_page_indices + b * out_stride;
int32_t* __restrict__ out_raw_row = out_raw_indices == nullptr ? nullptr : out_raw_indices + b * out_stride;
int64_t seq_len = load_index_value(seq_lens, b);
seq_len = std::max<int64_t>(0, std::min<int64_t>(seq_len, max_seq_len));
const int64_t valid_topk = std::min<int64_t>(seq_len, kC4Topk);
auto store_slot = [&](int64_t slot, int32_t raw_index) {
if (raw_index < 0) {
out_page_row[slot] = -1;
if (out_raw_row != nullptr) {
out_raw_row[slot] = -1;
}
return;
}
const int64_t page_idx = static_cast<int64_t>(raw_index) >> page_bits;
const int64_t offset_in_page = static_cast<int64_t>(raw_index) & page_mask;
const int64_t physical_page = load_index_value(page_table_row, page_idx);
out_page_row[slot] = static_cast<int32_t>((physical_page << page_bits) | offset_in_page);
if (out_raw_row != nullptr) {
out_raw_row[slot] = raw_index;
}
};
if (seq_len <= kC4Topk) {
for (int64_t i = 0; i < valid_topk; ++i) {
store_slot(i, static_cast<int32_t>(i));
}
for (int64_t i = valid_topk; i < kC4Topk; ++i) {
store_slot(i, -1);
}
continue;
}
for (int64_t i = 0; i < kC4Topk; ++i) {
heap[i] = {scores_row[i], static_cast<int32_t>(i)};
}
std::make_heap(heap.begin(), heap.end(), TopKTransformMinHeapCmp());
for (int64_t i = kC4Topk; i < seq_len; ++i) {
const float score = scores_row[i];
const TopKTransformElem& current_min = heap.front();
if (score > current_min.score ||
(score == current_min.score && static_cast<int32_t>(i) < current_min.index)) {
std::pop_heap(heap.begin(), heap.end(), TopKTransformMinHeapCmp());
heap.back() = {score, static_cast<int32_t>(i)};
std::push_heap(heap.begin(), heap.end(), TopKTransformMinHeapCmp());
}
}
for (int64_t i = 0; i < kC4Topk; ++i) {
store_slot(i, heap[i].index);
}
}
});
}
} // namespace
void topk_transform_512_cpu(
at::Tensor& scores,
at::Tensor& seq_lens,
at::Tensor& page_tables,
at::Tensor& out_page_indices,
int64_t page_size,
const std::optional<at::Tensor>& out_raw_indices) {
CHECK_INPUT(scores);
CHECK_INPUT(seq_lens);
CHECK_INPUT(page_tables);
CHECK_INPUT(out_page_indices);
TORCH_CHECK(scores.scalar_type() == at::kFloat, "scores must be float32");
TORCH_CHECK(out_page_indices.scalar_type() == at::kInt, "out_page_indices must be int32");
TORCH_CHECK(scores.dim() == 2, "scores must be a 2D tensor");
TORCH_CHECK(seq_lens.dim() == 1, "seq_lens must be a 1D tensor");
TORCH_CHECK(page_tables.dim() == 2, "page_tables must be a 2D tensor");
TORCH_CHECK(out_page_indices.dim() == 2, "out_page_indices must be a 2D tensor");
const int64_t batch_size = scores.size(0);
const int64_t max_seq_len = scores.size(1);
TORCH_CHECK(seq_lens.size(0) == batch_size, "seq_lens row count must match scores");
TORCH_CHECK(page_tables.size(0) == batch_size, "page_tables row count must match scores");
TORCH_CHECK(out_page_indices.size(0) == batch_size, "out_page_indices row count must match scores");
TORCH_CHECK(out_page_indices.size(1) >= kC4Topk, "out_page_indices must have at least 512 columns");
int32_t* raw_ptr = nullptr;
if (out_raw_indices.has_value()) {
at::Tensor raw = out_raw_indices.value();
CHECK_INPUT(raw);
TORCH_CHECK(raw.scalar_type() == at::kInt, "out_raw_indices must be int32");
TORCH_CHECK(raw.dim() == 2, "out_raw_indices must be a 2D tensor");
TORCH_CHECK(raw.sizes() == out_page_indices.sizes(), "out_raw_indices shape must match out_page_indices");
raw_ptr = raw.data_ptr<int32_t>();
}
if (batch_size == 0) {
return;
}
auto launch_with_seq_type = [&](auto seq_zero) {
using seq_t = decltype(seq_zero);
if (page_tables.scalar_type() == at::kInt) {
topk_transform_512_cpu_kernel_impl<seq_t, int32_t>(
scores.data_ptr<float>(), seq_lens.data_ptr<seq_t>(), page_tables.data_ptr<int32_t>(),
out_page_indices.data_ptr<int32_t>(), raw_ptr, batch_size, max_seq_len, page_tables.stride(0),
out_page_indices.stride(0), page_size);
} else if (page_tables.scalar_type() == at::kLong) {
topk_transform_512_cpu_kernel_impl<seq_t, int64_t>(
scores.data_ptr<float>(), seq_lens.data_ptr<seq_t>(), page_tables.data_ptr<int64_t>(),
out_page_indices.data_ptr<int32_t>(), raw_ptr, batch_size, max_seq_len, page_tables.stride(0),
out_page_indices.stride(0), page_size);
} else {
TORCH_CHECK(false, "page_tables must be int32 or int64");
}
};
if (seq_lens.scalar_type() == at::kInt) {
launch_with_seq_type(int32_t{0});
} else if (seq_lens.scalar_type() == at::kLong) {
launch_with_seq_type(int64_t{0});
} else {
TORCH_CHECK(false, "seq_lens must be int32 or int64");
}
}
+213 -16
View File
@@ -66,18 +66,18 @@ at::Tensor convert_weight_packed(at::Tensor& weight);
at::Tensor convert_scale_packed(at::Tensor& scale);
at::Tensor fused_experts_cpu(
at::Tensor& hidden_states, at::Tensor& w1, at::Tensor& w2,
at::Tensor& topk_weights, at::Tensor& topk_ids, bool inplace,
int64_t moe_comp_method, const std::optional<at::Tensor>& w1_scale,
const std::optional<at::Tensor>& w2_scale,
const std::optional<at::Tensor>& w1_zero,
const std::optional<at::Tensor>& w2_zero,
const std::optional<std::vector<int64_t>> block_size,
const std::optional<at::Tensor>& w1_bias,
const std::optional<at::Tensor>& w2_bias,
const std::optional<double>& alpha, const std::optional<double>& limit,
bool is_vnni);
void fused_experts_cpu(at::Tensor& out, at::Tensor& hidden_states,
at::Tensor& w1, at::Tensor& w2, at::Tensor& topk_weights,
at::Tensor& topk_ids, int64_t moe_comp_method,
const std::optional<at::Tensor>& w1_scale,
const std::optional<at::Tensor>& w2_scale,
const std::optional<at::Tensor>& w1_zero,
const std::optional<at::Tensor>& w2_zero,
const std::optional<std::vector<int64_t>> block_size,
const std::optional<at::Tensor>& w1_bias,
const std::optional<at::Tensor>& w2_bias,
const std::optional<double>& alpha,
const std::optional<double>& limit, bool is_vnni);
at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2,
at::Tensor& scales2,
@@ -124,6 +124,99 @@ void concat_and_cache_mla_cpu(const at::Tensor& kv_c_normed,
const at::Tensor& k_pe, at::Tensor& kv_cache,
const at::Tensor& slot_mapping);
// Adapted from sglang: DeepSeek-V4 mHC gating kernels
std::tuple<at::Tensor, at::Tensor, at::Tensor> hc_pre_fused_cpu(
at::Tensor& x, at::Tensor& hc_fn, at::Tensor& hc_scale, at::Tensor& hc_base,
int64_t hc_mult, int64_t sinkhorn_iters, double rms_eps, double hc_eps);
at::Tensor hc_post_fused_cpu(at::Tensor& x, at::Tensor& residual,
at::Tensor& post, at::Tensor& comb);
at::Tensor hc_head_fused_cpu(at::Tensor& x, at::Tensor& hc_fn,
at::Tensor& hc_scale, at::Tensor& hc_base,
double hc_eps, double norm_eps);
// Adapted from sglang: DeepSeek-V4 fp8_ds_mla cache-write kernel
at::Tensor fused_qnorm_rope_kv_insert_cpu(at::Tensor& q, at::Tensor& kv,
at::Tensor& positions,
at::Tensor& swa_kv_cache_2d,
at::Tensor& slot_mapping,
at::Tensor& cos_sin_cache,
int64_t q_head_padded, double eps,
int64_t cache_block_size);
// vLLM-native: DeepSeek-V4 sparse MQA attention kernel for the fp8_ds_mla
// cache layout (SWA window + optional compressed top-k index sets).
void flash_mla_with_kvcache_cpu(
at::Tensor& out, at::Tensor& q, at::Tensor& window_cache_2d,
at::Tensor& window_slots, int64_t window_block_size,
at::Tensor& compressed_cache_2d, at::Tensor& compressed_slots,
int64_t compressed_block_size, at::Tensor& attn_sink, double scale);
// Adapted from sglang: DeepSeek-V4 compressor state-cache write + fused
// compress+RMSNorm+quant+RoPE+store kernels.
void save_partial_states_cpu(at::Tensor& kv, at::Tensor& score, at::Tensor& ape,
at::Tensor& positions, at::Tensor& state_cache,
at::Tensor& slot_mapping);
void compress_norm_rope_store_cpu(
at::Tensor& state_cache, at::Tensor& gather_slots, at::Tensor& positions,
at::Tensor& kv_slot_mapping, at::Tensor& rms_norm_weight,
double rms_norm_eps, at::Tensor& cos_sin_cache, at::Tensor& kv_cache_2d,
int64_t kv_cache_block_size, int64_t compress_ratio);
// head_dim=128 indexer-compressor variant of the above (single fp8 quant
// block + raw fp32 scale -- matches the paged indexer K-cache layout
// fp8_paged_mqa_logits_cpu/topk_transform_512_cpu read).
void compress_norm_rope_store_indexer_cpu(
at::Tensor& state_cache, at::Tensor& gather_slots, at::Tensor& positions,
at::Tensor& kv_slot_mapping, at::Tensor& rms_norm_weight,
double rms_norm_eps, at::Tensor& cos_sin_cache, at::Tensor& kv_cache_2d,
int64_t kv_cache_block_size, int64_t compress_ratio);
// Adapted from sglang: DeepSeek-V4 sparse indexer paged MQA-logits +
// top-512 transform kernels (DECODE path -- reads the paged K-cache
// directly via `page_table`, whole batch in one call).
at::Tensor fp8_paged_mqa_logits_cpu(at::Tensor& q_fp8, at::Tensor& kvcache_fp8,
at::Tensor& weight, at::Tensor& seq_lens,
at::Tensor& page_table, int64_t block_size,
int64_t max_seq_len);
void topk_transform_512_cpu(at::Tensor& scores, at::Tensor& seq_lens,
at::Tensor& page_tables,
at::Tensor& out_page_indices, int64_t page_size,
const std::optional<at::Tensor>& out_raw_indices);
// Adapted from sglang: DeepSeek-V4 MoE routing kernels (flat biased top-k and
// hash-routed-layer top-k).
std::tuple<at::Tensor, at::Tensor> biased_topk_cpu(
at::Tensor& hidden_states, at::Tensor& gating_output,
at::Tensor& correction_bias, int64_t topk, bool renormalize,
std::string scoring_func, int64_t num_fused_shared_experts,
std::optional<double> routed_scaling_factor,
bool apply_routed_scaling_factor_on_output);
std::tuple<at::Tensor, at::Tensor> hash_topk_cpu(
at::Tensor& gating_output, at::Tensor& tid2eid, int64_t topk,
std::string scoring_func, int64_t num_fused_shared_experts,
int64_t num_experts, double routed_scaling_factor);
// vLLM-native: DeepSeek-V4 sparse indexer Q-side RoPE + FP8 quant kernel
// (fp8 path only; MXFP4 stays on triton-cpu).
void fused_indexer_q_rope_quant_cpu(at::Tensor& positions, at::Tensor& index_q,
at::Tensor& index_q_cos_sin_cache,
at::Tensor& index_q_fp8,
at::Tensor& index_weights,
double index_weights_softmax_scale,
double index_weights_head_scale,
at::Tensor& index_weights_out);
// vLLM-native: inverse GPT-J RoPE for DeepSeek-V4 CPU attention's `_o_proj`
// output de-rotation.
at::Tensor inverse_gptj_rope_o_proj_cpu(at::Tensor& o, at::Tensor& positions,
at::Tensor& cos_sin_cache,
int64_t rope_dim);
// Adapted from sglang: INT4 W4A8 kernels
std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_weight_packed_scale_zp(
at::Tensor qweight, // awq: (*, K, N / 8) || gptq: (*, K / 8, N) , int32
@@ -515,13 +608,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
ops.def("convert_scale_packed(Tensor! scale) -> Tensor");
ops.impl("convert_scale_packed", torch::kCPU, &convert_scale_packed);
ops.def(
"fused_experts_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor "
"topk_weights, Tensor topk_ids, bool "
"inplace, int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
"fused_experts_cpu(Tensor(a0!) out, Tensor hidden_states, Tensor w1, "
"Tensor w2, Tensor topk_weights, Tensor topk_ids, "
"int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
"Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, "
"Tensor? w1_bias, Tensor? w2_bias, float? alpha, float? limit, "
"bool is_vnni) -> "
"Tensor");
"()");
ops.impl("fused_experts_cpu", torch::kCPU, &fused_experts_cpu);
ops.def(
"int8_scaled_mm_with_quant(Tensor mat1, Tensor mat2, Tensor scales2, "
@@ -583,6 +676,110 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"concat_and_cache_mla_cpu(Tensor kv_c_normed, Tensor k_pe, "
"Tensor(a!) kv_cache, Tensor slot_mapping) -> ()");
ops.impl("concat_and_cache_mla_cpu", torch::kCPU, &concat_and_cache_mla_cpu);
// Adapted from sglang: DeepSeek-V4 mHC gating kernels
ops.def(
"hc_pre_fused_cpu(Tensor x, Tensor hc_fn, Tensor hc_scale, "
"Tensor hc_base, int hc_mult, int sinkhorn_iters, float rms_eps, "
"float hc_eps) -> (Tensor, Tensor, Tensor)");
ops.impl("hc_pre_fused_cpu", torch::kCPU, &hc_pre_fused_cpu);
ops.def(
"hc_post_fused_cpu(Tensor x, Tensor residual, Tensor post, "
"Tensor comb) -> Tensor");
ops.impl("hc_post_fused_cpu", torch::kCPU, &hc_post_fused_cpu);
ops.def(
"hc_head_fused_cpu(Tensor x, Tensor hc_fn, Tensor "
"hc_scale, Tensor hc_base, float hc_eps, float norm_eps) -> "
"Tensor");
ops.impl("hc_head_fused_cpu", torch::kCPU, &hc_head_fused_cpu);
// Adapted from sglang: DeepSeek-V4 fp8_ds_mla cache-write kernel
ops.def(
"fused_qnorm_rope_kv_insert_cpu(Tensor q, Tensor kv, Tensor positions, "
"Tensor(a!) swa_kv_cache_2d, Tensor slot_mapping, Tensor "
"cos_sin_cache, int q_head_padded, float eps, int cache_block_size) "
"-> Tensor");
ops.impl("fused_qnorm_rope_kv_insert_cpu", torch::kCPU,
&fused_qnorm_rope_kv_insert_cpu);
// vLLM-native: DeepSeek-V4 sparse MQA attention kernel
ops.def(
"flash_mla_with_kvcache_cpu(Tensor(a!) out, Tensor q, Tensor "
"window_cache_2d, Tensor window_slots, int window_block_size, Tensor "
"compressed_cache_2d, Tensor compressed_slots, int "
"compressed_block_size, Tensor attn_sink, float scale) -> ()");
ops.impl("flash_mla_with_kvcache_cpu", torch::kCPU,
&flash_mla_with_kvcache_cpu);
// Adapted from sglang: DeepSeek-V4 compressor kernels
ops.def(
"save_partial_states_cpu(Tensor kv, Tensor score, Tensor ape, "
"Tensor positions, Tensor(a!) state_cache, Tensor slot_mapping) "
"-> ()");
ops.impl("save_partial_states_cpu", torch::kCPU, &save_partial_states_cpu);
ops.def(
"compress_norm_rope_store_cpu(Tensor state_cache, Tensor "
"gather_slots, Tensor positions, Tensor kv_slot_mapping, Tensor "
"rms_norm_weight, float rms_norm_eps, Tensor cos_sin_cache, "
"Tensor(a!) kv_cache_2d, int kv_cache_block_size, int compress_ratio) "
"-> ()");
ops.impl("compress_norm_rope_store_cpu", torch::kCPU,
&compress_norm_rope_store_cpu);
ops.def(
"compress_norm_rope_store_indexer_cpu(Tensor state_cache, Tensor "
"gather_slots, Tensor positions, Tensor kv_slot_mapping, Tensor "
"rms_norm_weight, float rms_norm_eps, Tensor cos_sin_cache, "
"Tensor(a!) kv_cache_2d, int kv_cache_block_size, int compress_ratio) "
"-> ()");
ops.impl("compress_norm_rope_store_indexer_cpu", torch::kCPU,
&compress_norm_rope_store_indexer_cpu);
// Adapted from sglang: DeepSeek-V4 sparse indexer paged MQA-logits kernel
// (DECODE path)
ops.def(
"fp8_paged_mqa_logits_cpu(Tensor q_fp8, Tensor kvcache_fp8, Tensor "
"weight, Tensor seq_lens, Tensor page_table, int block_size, int "
"max_seq_len) -> Tensor");
ops.impl("fp8_paged_mqa_logits_cpu", torch::kCPU, &fp8_paged_mqa_logits_cpu);
ops.def(
"topk_transform_512_cpu(Tensor scores, Tensor seq_lens, Tensor "
"page_tables, Tensor(a!) out_page_indices, int page_size, Tensor(b!)? "
"out_raw_indices) -> ()");
ops.impl("topk_transform_512_cpu", torch::kCPU, &topk_transform_512_cpu);
ops.def(
"fused_indexer_q_rope_quant_cpu(Tensor positions, Tensor index_q, "
"Tensor index_q_cos_sin_cache, Tensor(a!) index_q_fp8, Tensor "
"index_weights, float index_weights_softmax_scale, float "
"index_weights_head_scale, Tensor(b!) index_weights_out) -> ()");
ops.impl("fused_indexer_q_rope_quant_cpu", torch::kCPU,
&fused_indexer_q_rope_quant_cpu);
// Adapted from sglang: DeepSeek-V4 MoE routing kernels
ops.def(
"biased_topk_cpu(Tensor hidden_states, Tensor gating_output, Tensor "
"correction_bias, int topk, bool renormalize, str scoring_func, int "
"num_fused_shared_experts, float? routed_scaling_factor, bool "
"apply_routed_scaling_factor_on_output) -> (Tensor, Tensor)");
ops.impl("biased_topk_cpu", torch::kCPU, &biased_topk_cpu);
ops.def(
"hash_topk_cpu(Tensor gating_output, Tensor tid2eid, int topk, str "
"scoring_func, int num_fused_shared_experts, int num_experts, float "
"routed_scaling_factor) -> (Tensor, Tensor)");
ops.impl("hash_topk_cpu", torch::kCPU, &hash_topk_cpu);
// vLLM-native: inverse GPT-J RoPE for _o_proj's output de-rotation
ops.def(
"inverse_gptj_rope_o_proj_cpu(Tensor o, Tensor positions, Tensor "
"cos_sin_cache, int rope_dim) -> Tensor");
ops.impl("inverse_gptj_rope_o_proj_cpu", torch::kCPU,
&inverse_gptj_rope_o_proj_cpu);
#endif
#if (defined(__AVX512BF16__) && defined(__AVX512F__) && \