[Model] Add DeepSeek-V4 CPU backend (#55355)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -343,6 +343,36 @@ inline void parallel_2d(int m, int n, const func_t& f) {
|
||||
#endif
|
||||
}
|
||||
|
||||
// Like parallel_2d but with explicit nth_m x nth_n decomposition.
|
||||
// Caller is responsible for choosing nth_m and nth_n.
|
||||
//
|
||||
// Cherry-picked from a newer sglang commit than this file's last full sync
|
||||
// (needed by mhc.cpp's phase-1 tiling); not part of a full common.h resync.
|
||||
template <typename func_t>
|
||||
inline void parallel_2d_tiled(int m, int n, int nth_m, int nth_n, const func_t& f) {
|
||||
const int nth = nth_m * nth_n;
|
||||
#if defined(_OPENMP)
|
||||
#pragma omp parallel num_threads(nth)
|
||||
{
|
||||
int ith = omp_get_thread_num();
|
||||
int ith_m = ith / nth_n;
|
||||
int ith_n = ith % nth_n;
|
||||
|
||||
int thread_block_m = div_up(m, nth_m);
|
||||
int thread_block_n = div_up(n, nth_n);
|
||||
|
||||
int begin_m = std::min(ith_m * thread_block_m, m);
|
||||
int end_m = std::min(begin_m + thread_block_m, m);
|
||||
int begin_n = std::min(ith_n * thread_block_n, n);
|
||||
int end_n = std::min(begin_n + thread_block_n, n);
|
||||
|
||||
f(begin_m, end_m, begin_n, end_n);
|
||||
}
|
||||
#else
|
||||
f(0, m, 0, n);
|
||||
#endif
|
||||
}
|
||||
|
||||
// limit max cache blocks
|
||||
// when we need to do pre-unpack for weights, e.g. fp8
|
||||
#define MAX_CACHE_BLOCK_SIZE 4
|
||||
|
||||
@@ -0,0 +1,747 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/blob/main/python/sglang/kernels/aot/csrc/cpu/compressor.cpp
|
||||
//
|
||||
// DeepSeek-V4 compressor: softmax-weighted pooling over a sliding window of
|
||||
// raw KV/score state, RMSNorm, FP8/UE8M0 quant (NoPE) + GPT-J RoPE (RoPE),
|
||||
// paged-insert into the fp8_ds_mla cache layout (see store_cache.cpp).
|
||||
// Covers both the head_dim=512 attention-compressor path
|
||||
// (`compress_norm_rope_store_cpu`) and the head_dim=128 indexer-compressor
|
||||
// path (`compress_norm_rope_store_indexer_cpu`, distinct cache layout --
|
||||
// see its own comment below).
|
||||
//
|
||||
// Differences from upstream:
|
||||
// - Upstream splits decode (`compress_decode_cpu`) from prefill/extend
|
||||
// (eager `compress_extend_separate`); this port stays unified across
|
||||
// decode/prefill/extend, one program per token, matching vLLM's own
|
||||
// GPU/Triton reference (`_fused_kv_compress_norm_rope_insert_sparse_attn`,
|
||||
// common/ops/fused_compress_quant_cache.py).
|
||||
// - Window-gather indices arrive pre-resolved to flat state-cache row ids
|
||||
// from the Python caller (mirroring flash_mla.cpp), not looked up via
|
||||
// block_table in-kernel.
|
||||
// - The state cache is pooled with the main KV cache's physical pages
|
||||
// (CompressorStateCache in compressor.py), so its per-block byte stride
|
||||
// is page-aligned, not `block_size * row_width` -- row ids are addressed
|
||||
// as (block_idx, pos_in_block) against the tensor's real block stride,
|
||||
// never by flattening to a 2D view.
|
||||
//
|
||||
// clang-format off
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
|
||||
#if !defined(CPU_CAPABILITY_AVX512)
|
||||
static_assert(false,
|
||||
"compressor.cpp is only ever compiled into the AVX512+AMX-flagged "
|
||||
"_C target (see cmake/cpu_extension.cmake's VLLM_EXT_SRC_SGL) and "
|
||||
"relies on that unconditionally -- CPU_CAPABILITY_AVX512 must be "
|
||||
"defined here.");
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
|
||||
constexpr int64_t kNopeDim = 448;
|
||||
constexpr int64_t kRopeDim = 64;
|
||||
constexpr int64_t kHeadDim = kNopeDim + kRopeDim;
|
||||
constexpr int64_t kQuantBlock = 64;
|
||||
constexpr int64_t kNumQuantBlocks = kNopeDim / kQuantBlock;
|
||||
constexpr int64_t kScaleBytesPerToken = kNumQuantBlocks + 1;
|
||||
constexpr int64_t kTokenDataBytes = kNopeDim + kRopeDim * 2;
|
||||
constexpr float kFp8Max = 448.0f;
|
||||
constexpr float kFp8Min = -448.0f;
|
||||
constexpr float kQuantEps = 1e-4f;
|
||||
|
||||
// In-place interleaved (GPT-J-style) RoPE: x[2i]/x[2i+1] are one (even, odd)
|
||||
// pair, cos[i]/sin[i] its angle. Adapted from SGLang's `rope.cpp`
|
||||
// (`apply_rotary_emb_row`), which avoids gather/scatter (slow on AVX512) by
|
||||
// broadcasting cos/sin across each pair via in-register permutes instead:
|
||||
// `_mm512_permute_ps(xv, 0xB1)` swaps the (even, odd) lanes within every
|
||||
// 2-float pair in one shuffle, and cos/sin -- loaded as 8 contiguous values
|
||||
// -- are duplicated pairwise into the same layout via `_mm512_permutexvar_ps`.
|
||||
// SGLang's version reads cos/sin pre-interleaved with x (`[c0,s0,c1,s1,...]`)
|
||||
// since its freqs tensor is laid out that way; ours are two separate flat
|
||||
// arrays (`cos[i]`, `sin[i]`, matching store_cache.cpp's convention), hence
|
||||
// the extra broadcast-duplicate permute here instead of a plain load.
|
||||
// Both call sites in this file pass half_dim=32 (kRopeDim/2, kIndexerRopeDim/2
|
||||
// are both 64/2), an exact multiple of 8, so this file's build (always AVX512,
|
||||
// see file header) never needs a scalar remainder.
|
||||
inline void apply_gptj_rope_inplace(float* __restrict__ x, const float* __restrict__ cos, const float* __restrict__ sin, int64_t half_dim) {
|
||||
// Duplicates each of 8 source lanes into 2 adjacent output lanes:
|
||||
// [c0,c1,...,c7] -> [c0,c0,c1,c1,...,c7,c7].
|
||||
const __m512i dup_idx = _mm512_set_epi32(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0);
|
||||
// Forward GPT-J rotation negates the sin broadcast on the even (real)
|
||||
// lane of each pair; the odd (imag) lane keeps sin's sign.
|
||||
const __m512 sign_mask = _mm512_castsi512_ps(_mm512_set_epi32(
|
||||
0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000,
|
||||
0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000));
|
||||
for (int64_t i = 0; i + 8 <= half_dim; i += 8) {
|
||||
__m512 xv = _mm512_loadu_ps(x + 2 * i);
|
||||
__m512 xv_swapped = _mm512_permute_ps(xv, 0xB1);
|
||||
__m512 cos_b = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(cos + i)));
|
||||
__m512 sin_b = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(sin + i)));
|
||||
__m512 sin_signed = _mm512_xor_ps(sin_b, sign_mask);
|
||||
__m512 out = _mm512_fmadd_ps(xv, cos_b, _mm512_mul_ps(xv_swapped, sin_signed));
|
||||
_mm512_storeu_ps(x + 2 * i, out);
|
||||
}
|
||||
}
|
||||
|
||||
// AVX512 tile-quantization, adapted verbatim from SGLang's `store_cache.cpp`
|
||||
// (`quant_to_nope_fp8_rope_bf16_pack_cpu`'s
|
||||
// `cvt_fp32x16_to_fp8x16`/`quantize_tile_avx512`) -- SGLang itself keeps
|
||||
// `compress_decode_cpu` fp32-output-only and calls this quant step as a
|
||||
// separate op; vLLM's compressor kernel stays fused (per this file's header
|
||||
// comment), so the same AVX512 tile-quant math is inlined here instead.
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
|
||||
// Vectorized float32x16 -> fp8_e4m3fn x16 using AVX512 bit manipulation.
|
||||
// Implements RNE (round-to-nearest-even). Flushes subnormal fp8 to zero.
|
||||
// fp8_e4m3fn: sign(1), exp(4, bias=7), mant(3), no inf/nan special values.
|
||||
// Normal: exp_field in [1..15], value = 2^(exp-7) * (1 + mant/8)
|
||||
// Input assumed clamped to [-448, 448].
|
||||
inline __m128i cvt_fp32x16_to_fp8x16(__m512 input) {
|
||||
__m512i bits = _mm512_castps_si512(input);
|
||||
|
||||
__m512i signs = _mm512_srli_epi32(_mm512_and_si512(bits, _mm512_set1_epi32(0x80000000)), 24);
|
||||
__m512i abs_bits = _mm512_and_si512(bits, _mm512_set1_epi32(0x7FFFFFFF));
|
||||
__m512i f32_exp = _mm512_srli_epi32(abs_bits, 23);
|
||||
__m512i f32_mant = _mm512_and_si512(abs_bits, _mm512_set1_epi32(0x7FFFFF));
|
||||
__m512i fp8_exp = _mm512_sub_epi32(f32_exp, _mm512_set1_epi32(120));
|
||||
__m512i mant3 = _mm512_srli_epi32(f32_mant, 20);
|
||||
|
||||
__m512i round_bit = _mm512_and_si512(_mm512_srli_epi32(f32_mant, 19), _mm512_set1_epi32(1));
|
||||
__m512i sticky_bits = _mm512_and_si512(f32_mant, _mm512_set1_epi32(0x7FFFF));
|
||||
__mmask16 has_sticky = _mm512_cmpneq_epi32_mask(sticky_bits, _mm512_setzero_si512());
|
||||
|
||||
__m512i lsb = _mm512_and_si512(mant3, _mm512_set1_epi32(1));
|
||||
__m512i sticky_or_lsb = _mm512_or_si512(_mm512_maskz_mov_epi32(has_sticky, _mm512_set1_epi32(1)), lsb);
|
||||
__m512i do_round = _mm512_and_si512(round_bit, sticky_or_lsb);
|
||||
mant3 = _mm512_add_epi32(mant3, do_round);
|
||||
|
||||
__mmask16 mant_ovf = _mm512_cmpeq_epi32_mask(mant3, _mm512_set1_epi32(8));
|
||||
mant3 = _mm512_mask_mov_epi32(mant3, mant_ovf, _mm512_setzero_si512());
|
||||
fp8_exp = _mm512_mask_add_epi32(fp8_exp, mant_ovf, fp8_exp, _mm512_set1_epi32(1));
|
||||
|
||||
fp8_exp = _mm512_min_epi32(fp8_exp, _mm512_set1_epi32(15));
|
||||
|
||||
__m512i result = _mm512_or_si512(_mm512_slli_epi32(fp8_exp, 3), mant3);
|
||||
|
||||
__mmask16 is_subnormal_or_zero = _mm512_cmplt_epi32_mask(f32_exp, _mm512_set1_epi32(121));
|
||||
result = _mm512_mask_mov_epi32(result, is_subnormal_or_zero, _mm512_setzero_si512());
|
||||
|
||||
result = _mm512_or_si512(result, signs);
|
||||
|
||||
return _mm512_cvtepi32_epi8(result);
|
||||
}
|
||||
|
||||
// Process one 64-wide tile: find amax, compute a shared power-of-two scale,
|
||||
// quantize to fp8. Returns the UE8M0 scale byte for the tile.
|
||||
inline uint8_t quantize_tile_avx512(const at::BFloat16* __restrict__ src, uint8_t* __restrict__ dst) {
|
||||
__m256i bf16_0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src));
|
||||
__m256i bf16_1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 16));
|
||||
__m256i bf16_2 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 32));
|
||||
__m256i bf16_3 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 48));
|
||||
|
||||
__m512 f0 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_0), 16));
|
||||
__m512 f1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_1), 16));
|
||||
__m512 f2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_2), 16));
|
||||
__m512 f3 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_3), 16));
|
||||
|
||||
const __m512i abs_mask_i = _mm512_set1_epi32(0x7FFFFFFF);
|
||||
__m512 abs0 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f0), abs_mask_i));
|
||||
__m512 abs1 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f1), abs_mask_i));
|
||||
__m512 abs2 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f2), abs_mask_i));
|
||||
__m512 abs3 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f3), abs_mask_i));
|
||||
|
||||
__m512 max01 = _mm512_max_ps(abs0, abs1);
|
||||
__m512 max23 = _mm512_max_ps(abs2, abs3);
|
||||
__m512 max0123 = _mm512_max_ps(max01, max23);
|
||||
float amax = _mm512_reduce_max_ps(max0123);
|
||||
|
||||
float scale = std::max(amax / kFp8Max, kQuantEps);
|
||||
float ceil_log2 = std::ceil(std::log2(scale));
|
||||
float scale_pow2 = std::exp2(ceil_log2);
|
||||
float scale_inv = 1.0f / scale_pow2;
|
||||
|
||||
int exponent = static_cast<int>(ceil_log2);
|
||||
uint8_t scale_uint8 = static_cast<uint8_t>(exponent + 127);
|
||||
|
||||
__m512 vinv = _mm512_set1_ps(scale_inv);
|
||||
__m512 s0 = _mm512_mul_ps(f0, vinv);
|
||||
__m512 s1 = _mm512_mul_ps(f1, vinv);
|
||||
__m512 s2 = _mm512_mul_ps(f2, vinv);
|
||||
__m512 s3 = _mm512_mul_ps(f3, vinv);
|
||||
|
||||
__m512 vmax = _mm512_set1_ps(kFp8Max);
|
||||
__m512 vmin = _mm512_set1_ps(kFp8Min);
|
||||
s0 = _mm512_max_ps(_mm512_min_ps(s0, vmax), vmin);
|
||||
s1 = _mm512_max_ps(_mm512_min_ps(s1, vmax), vmin);
|
||||
s2 = _mm512_max_ps(_mm512_min_ps(s2, vmax), vmin);
|
||||
s3 = _mm512_max_ps(_mm512_min_ps(s3, vmax), vmin);
|
||||
|
||||
__m128i fp8_0 = cvt_fp32x16_to_fp8x16(s0);
|
||||
__m128i fp8_1 = cvt_fp32x16_to_fp8x16(s1);
|
||||
__m128i fp8_2 = cvt_fp32x16_to_fp8x16(s2);
|
||||
__m128i fp8_3 = cvt_fp32x16_to_fp8x16(s3);
|
||||
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), fp8_0);
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), fp8_1);
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), fp8_2);
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), fp8_3);
|
||||
|
||||
// Patch subnormal-magnitude lanes: `cvt_fp32x16_to_fp8x16` flushes
|
||||
// anything below the smallest fp8_e4m3fn normal (2^-6) to zero, but
|
||||
// vLLM's dequant path relies on `at::Float8_e4m3fn`'s real subnormal
|
||||
// encoding -- redo just those rare lanes with the exact scalar
|
||||
// conversion to stay consistent with the read side.
|
||||
alignas(64) float scaled[kQuantBlock];
|
||||
_mm512_store_ps(scaled, s0);
|
||||
_mm512_store_ps(scaled + 16, s1);
|
||||
_mm512_store_ps(scaled + 32, s2);
|
||||
_mm512_store_ps(scaled + 48, s3);
|
||||
for (int64_t j = 0; j < kQuantBlock; ++j) {
|
||||
const float av = std::abs(scaled[j]);
|
||||
if (av != 0.f && av < 0x1p-6f) {
|
||||
dst[j] = at::Float8_e4m3fn(scaled[j]).x;
|
||||
}
|
||||
}
|
||||
|
||||
return scale_uint8;
|
||||
}
|
||||
|
||||
#endif // CPU_CAPABILITY_AVX512
|
||||
|
||||
void save_partial_states_impl(
|
||||
float* __restrict__ state_cache, // [num_blocks, block_stride]
|
||||
int64_t block_stride, // element stride between blocks
|
||||
int64_t block_size, // rows (slots) per block
|
||||
const float* __restrict__ kv, // [num_tokens, state_width]
|
||||
int64_t kv_row_stride, // element stride between kv rows
|
||||
const float* __restrict__ score, // [num_tokens, state_width]
|
||||
int64_t score_row_stride, // element stride between score rows
|
||||
const float* __restrict__ ape, // [compress_ratio, state_width]
|
||||
const int64_t* __restrict__ positions, // [num_tokens]
|
||||
const int64_t* __restrict__ slot_mapping, // [num_tokens]
|
||||
int64_t state_width,
|
||||
int64_t compress_ratio,
|
||||
int64_t num_tokens) {
|
||||
const int64_t row_width = 2 * state_width;
|
||||
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t t = begin; t < end; ++t) {
|
||||
const int64_t slot_id = slot_mapping[t];
|
||||
if (slot_id < 0) continue;
|
||||
|
||||
const int64_t block_idx = slot_id / block_size;
|
||||
const int64_t pos_in_block = slot_id % block_size;
|
||||
float* const row = state_cache + block_idx * block_stride + pos_in_block * row_width;
|
||||
const float* const kv_t = kv + t * kv_row_stride;
|
||||
// state_width is always a multiple of 16 for this model (512/1024 for
|
||||
// the main compressor, 128/256 for the indexer compressor).
|
||||
for (int64_t i = 0; i + 16 <= state_width; i += 16) {
|
||||
fVec::loadu(kv_t + i).store(row + i);
|
||||
}
|
||||
|
||||
const int64_t position = positions[t];
|
||||
const float* const ape_row = ape + (position % compress_ratio) * state_width;
|
||||
const float* const score_t = score + t * score_row_stride;
|
||||
float* const score_out = row + state_width;
|
||||
for (int64_t i = 0; i + 16 <= state_width; i += 16) {
|
||||
(fVec::loadu(score_t + i) + fVec::loadu(ape_row + i)).store(score_out + i);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void compress_norm_rope_store_impl(
|
||||
const float* __restrict__ state_cache, // [num_blocks, state_block_stride]
|
||||
int64_t state_block_stride, // element stride between state blocks
|
||||
int64_t state_block_size, // rows (slots) per state block
|
||||
const int64_t* __restrict__ gather_slots, // [num_tokens, window], -1 = invalid
|
||||
const int64_t* __restrict__ positions, // [num_tokens]
|
||||
const int64_t* __restrict__ kv_slot_mapping, // [num_tokens], -1 = skip
|
||||
const float* __restrict__ rms_norm_weight, // [kHeadDim]
|
||||
float rms_norm_eps,
|
||||
const float* __restrict__ cos_sin_cache, // [max_pos, kRopeDim]
|
||||
uint8_t* __restrict__ kv_cache, // [num_blocks, cache_row_stride]
|
||||
int64_t cache_row_stride,
|
||||
int64_t kv_cache_block_size,
|
||||
int64_t compress_ratio,
|
||||
int64_t window,
|
||||
int64_t state_width,
|
||||
int64_t num_tokens) {
|
||||
const int64_t rope_half = kRopeDim / 2;
|
||||
const int64_t state_row_width = 2 * state_width;
|
||||
|
||||
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
|
||||
std::array<float, kHeadDim> max_v;
|
||||
std::array<float, kHeadDim> denom;
|
||||
std::array<float, kHeadDim> acc;
|
||||
std::array<float, kHeadDim> normed;
|
||||
std::array<at::BFloat16, kNopeDim> nope_bf16;
|
||||
|
||||
for (int64_t t = begin; t < end; ++t) {
|
||||
const int64_t position = positions[t];
|
||||
if ((position + 1) % compress_ratio != 0) continue;
|
||||
const int64_t kv_slot = kv_slot_mapping[t];
|
||||
if (kv_slot < 0) continue;
|
||||
|
||||
std::fill(max_v.begin(), max_v.end(), -std::numeric_limits<float>::infinity());
|
||||
std::fill(denom.begin(), denom.end(), 0.f);
|
||||
std::fill(acc.begin(), acc.end(), 0.f);
|
||||
|
||||
// Per-channel (not per-position) softmax over the window: each of the
|
||||
// kHeadDim columns is normalized independently across the window axis.
|
||||
for (int64_t w = 0; w < window; ++w) {
|
||||
const int64_t slot = gather_slots[t * window + w];
|
||||
if (slot < 0) continue;
|
||||
const int64_t block_idx = slot / state_block_size;
|
||||
const int64_t pos_in_block = slot % state_block_size;
|
||||
const int64_t head_offset = (w >= compress_ratio) ? kHeadDim : 0;
|
||||
const float* const score_row = state_cache + block_idx * state_block_stride +
|
||||
pos_in_block * state_row_width + head_offset + state_width;
|
||||
for (int64_t d = 0; d < kHeadDim; d += 16) {
|
||||
__m512 mx = _mm512_loadu_ps(max_v.data() + d);
|
||||
__m512 sv = _mm512_loadu_ps(score_row + d);
|
||||
_mm512_storeu_ps(max_v.data() + d, _mm512_max_ps(mx, sv));
|
||||
}
|
||||
}
|
||||
for (int64_t w = 0; w < window; ++w) {
|
||||
const int64_t slot = gather_slots[t * window + w];
|
||||
if (slot < 0) continue;
|
||||
const int64_t block_idx = slot / state_block_size;
|
||||
const int64_t pos_in_block = slot % state_block_size;
|
||||
const int64_t head_offset = (w >= compress_ratio) ? kHeadDim : 0;
|
||||
const float* const row = state_cache + block_idx * state_block_stride +
|
||||
pos_in_block * state_row_width + head_offset;
|
||||
const float* const score_row = row + state_width;
|
||||
for (int64_t d = 0; d < kHeadDim; d += 16) {
|
||||
fVec e_v = (fVec::loadu(score_row + d) - fVec::loadu(max_v.data() + d)).exp();
|
||||
(fVec::loadu(denom.data() + d) + e_v).store(denom.data() + d);
|
||||
(fVec::loadu(acc.data() + d) + e_v * fVec::loadu(row + d)).store(acc.data() + d);
|
||||
}
|
||||
}
|
||||
|
||||
float sq_sum = 0.f;
|
||||
{
|
||||
fVec sq_sum_v(0.f);
|
||||
for (int64_t d = 0; d < kHeadDim; d += 16) {
|
||||
fVec v = fVec::loadu(acc.data() + d) / fVec::loadu(denom.data() + d);
|
||||
v.store(normed.data() + d);
|
||||
sq_sum_v = sq_sum_v + v * v;
|
||||
}
|
||||
sq_sum = vec_reduce_sum(sq_sum_v);
|
||||
}
|
||||
const float inv_rms = 1.0f / std::sqrt(sq_sum / static_cast<float>(kHeadDim) + rms_norm_eps);
|
||||
{
|
||||
fVec inv_rms_v(inv_rms);
|
||||
for (int64_t d = 0; d < kHeadDim; d += 16) {
|
||||
(fVec::loadu(normed.data() + d) * inv_rms_v * fVec::loadu(rms_norm_weight + d)).store(normed.data() + d);
|
||||
}
|
||||
}
|
||||
|
||||
// bf16 round-trip feeds only the NoPE FP8 quantization (RoPE rotates
|
||||
// the un-rounded `normed` below, then casts once to bf16 at store
|
||||
// time for the separate bf16 rope region).
|
||||
for (int64_t d = 0; d < kNopeDim; d += 16)
|
||||
store_from_float_ext<at::BFloat16>(nope_bf16.data() + d, fVec::loadu(normed.data() + d));
|
||||
|
||||
const int64_t compressed_pos = (position / compress_ratio) * compress_ratio;
|
||||
const float* const cos = cos_sin_cache + compressed_pos * kRopeDim;
|
||||
const float* const sin = cos + rope_half;
|
||||
apply_gptj_rope_inplace(normed.data() + kNopeDim, cos, sin, rope_half);
|
||||
|
||||
const int64_t block_idx = kv_slot / kv_cache_block_size;
|
||||
const int64_t pos_in_block = kv_slot % kv_cache_block_size;
|
||||
uint8_t* const block_base = kv_cache + block_idx * cache_row_stride;
|
||||
uint8_t* const token_fp8_ptr = block_base + pos_in_block * kTokenDataBytes;
|
||||
uint8_t* const token_bf16_ptr = token_fp8_ptr + kNopeDim;
|
||||
uint8_t* const token_scale_ptr =
|
||||
block_base + kv_cache_block_size * kTokenDataBytes + pos_in_block * kScaleBytesPerToken;
|
||||
|
||||
// AVX512 tile-quantization (SGLang's `quantize_tile_avx512`), one
|
||||
// 64-wide tile per iteration.
|
||||
for (int64_t qb = 0; qb < kNumQuantBlocks; ++qb) {
|
||||
const at::BFloat16* tile_src = nope_bf16.data() + qb * kQuantBlock;
|
||||
uint8_t* tile_dst = token_fp8_ptr + qb * kQuantBlock;
|
||||
token_scale_ptr[qb] = quantize_tile_avx512(tile_src, tile_dst);
|
||||
}
|
||||
token_scale_ptr[kNumQuantBlocks] = 0;
|
||||
|
||||
auto* const out_rope = reinterpret_cast<at::BFloat16*>(token_bf16_ptr);
|
||||
for (int64_t ri = 0; ri < kRopeDim; ri += 16)
|
||||
store_from_float_ext<at::BFloat16>(out_rope + ri, fVec::loadu(normed.data() + kNopeDim + ri));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Indexer compressor (head_dim=128): same softmax-pool + RMSNorm + GPT-J
|
||||
// RoPE prologue as the head=512 path above, but a different cache layout:
|
||||
// all 128 dims are FP8 in one combined quant block (no separate bf16 rope
|
||||
// region, single raw fp32 scale per token) instead of 7 UE8M0 blocks over
|
||||
// nope alone -- matching the paged indexer K-cache layout
|
||||
// fp8_paged_mqa_logits_cpu/topk_transform_512_cpu (paged_mqa_logits.cpp/
|
||||
// topk.cpp) read, and the GPU/Triton reference
|
||||
// (`_fused_kv_compress_norm_rope_insert_indexer_attn`,
|
||||
// common/ops/fused_compress_quant_cache.py) exactly.
|
||||
constexpr int64_t kIndexerHeadDim = 128;
|
||||
constexpr int64_t kIndexerRopeDim = 64;
|
||||
constexpr int64_t kIndexerNopeDim = kIndexerHeadDim - kIndexerRopeDim;
|
||||
constexpr int64_t kIndexerTokenBytes = kIndexerHeadDim;
|
||||
constexpr int64_t kIndexerScaleBytes = 4;
|
||||
|
||||
void compress_norm_rope_store_indexer_impl(
|
||||
const float* __restrict__ state_cache, // [num_blocks, state_block_stride]
|
||||
int64_t state_block_stride,
|
||||
int64_t state_block_size,
|
||||
const int64_t* __restrict__ gather_slots, // [num_tokens, window], -1 = invalid
|
||||
const int64_t* __restrict__ positions, // [num_tokens]
|
||||
const int64_t* __restrict__ kv_slot_mapping, // [num_tokens], -1 = skip
|
||||
const float* __restrict__ rms_norm_weight, // [kIndexerHeadDim]
|
||||
float rms_norm_eps,
|
||||
const float* __restrict__ cos_sin_cache, // [max_pos, kIndexerRopeDim]
|
||||
uint8_t* __restrict__ kv_cache, // [num_blocks, cache_row_stride]
|
||||
int64_t cache_row_stride,
|
||||
int64_t kv_cache_block_size,
|
||||
int64_t compress_ratio,
|
||||
int64_t window,
|
||||
int64_t state_width,
|
||||
int64_t num_tokens) {
|
||||
const int64_t rope_half = kIndexerRopeDim / 2;
|
||||
const int64_t state_row_width = 2 * state_width;
|
||||
|
||||
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
|
||||
std::array<float, kIndexerHeadDim> max_v;
|
||||
std::array<float, kIndexerHeadDim> denom;
|
||||
std::array<float, kIndexerHeadDim> acc;
|
||||
std::array<float, kIndexerHeadDim> normed;
|
||||
std::array<at::BFloat16, kIndexerHeadDim> quant_bf16;
|
||||
|
||||
for (int64_t t = begin; t < end; ++t) {
|
||||
const int64_t position = positions[t];
|
||||
if ((position + 1) % compress_ratio != 0) continue;
|
||||
const int64_t kv_slot = kv_slot_mapping[t];
|
||||
if (kv_slot < 0) continue;
|
||||
|
||||
std::fill(max_v.begin(), max_v.end(), -std::numeric_limits<float>::infinity());
|
||||
std::fill(denom.begin(), denom.end(), 0.f);
|
||||
std::fill(acc.begin(), acc.end(), 0.f);
|
||||
|
||||
for (int64_t w = 0; w < window; ++w) {
|
||||
const int64_t slot = gather_slots[t * window + w];
|
||||
if (slot < 0) continue;
|
||||
const int64_t block_idx = slot / state_block_size;
|
||||
const int64_t pos_in_block = slot % state_block_size;
|
||||
const int64_t head_offset = (w >= compress_ratio) ? kIndexerHeadDim : 0;
|
||||
const float* const score_row = state_cache + block_idx * state_block_stride +
|
||||
pos_in_block * state_row_width + head_offset + state_width;
|
||||
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
|
||||
__m512 mx = _mm512_loadu_ps(max_v.data() + d);
|
||||
__m512 sv = _mm512_loadu_ps(score_row + d);
|
||||
_mm512_storeu_ps(max_v.data() + d, _mm512_max_ps(mx, sv));
|
||||
}
|
||||
}
|
||||
for (int64_t w = 0; w < window; ++w) {
|
||||
const int64_t slot = gather_slots[t * window + w];
|
||||
if (slot < 0) continue;
|
||||
const int64_t block_idx = slot / state_block_size;
|
||||
const int64_t pos_in_block = slot % state_block_size;
|
||||
const int64_t head_offset = (w >= compress_ratio) ? kIndexerHeadDim : 0;
|
||||
const float* const row = state_cache + block_idx * state_block_stride +
|
||||
pos_in_block * state_row_width + head_offset;
|
||||
const float* const score_row = row + state_width;
|
||||
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
|
||||
fVec e_v = (fVec::loadu(score_row + d) - fVec::loadu(max_v.data() + d)).exp();
|
||||
(fVec::loadu(denom.data() + d) + e_v).store(denom.data() + d);
|
||||
(fVec::loadu(acc.data() + d) + e_v * fVec::loadu(row + d)).store(acc.data() + d);
|
||||
}
|
||||
}
|
||||
|
||||
float sq_sum = 0.f;
|
||||
{
|
||||
fVec sq_sum_v(0.f);
|
||||
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
|
||||
fVec v = fVec::loadu(acc.data() + d) / fVec::loadu(denom.data() + d);
|
||||
v.store(normed.data() + d);
|
||||
sq_sum_v = sq_sum_v + v * v;
|
||||
}
|
||||
sq_sum = vec_reduce_sum(sq_sum_v);
|
||||
}
|
||||
const float inv_rms = 1.0f / std::sqrt(sq_sum / static_cast<float>(kIndexerHeadDim) + rms_norm_eps);
|
||||
{
|
||||
fVec inv_rms_v(inv_rms);
|
||||
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
|
||||
(fVec::loadu(normed.data() + d) * inv_rms_v * fVec::loadu(rms_norm_weight + d)).store(normed.data() + d);
|
||||
}
|
||||
}
|
||||
|
||||
const int64_t compressed_pos = (position / compress_ratio) * compress_ratio;
|
||||
const float* const cos = cos_sin_cache + compressed_pos * kIndexerRopeDim;
|
||||
const float* const sin = cos + rope_half;
|
||||
apply_gptj_rope_inplace(normed.data() + kIndexerNopeDim, cos, sin, rope_half);
|
||||
|
||||
// Indexer path rounds the FULL post-RoPE vector through bf16 before
|
||||
// quantizing (single combined block); the head=512 path above instead
|
||||
// rounds only the NoPE region before RoPE (separate bf16 rope store).
|
||||
for (int64_t d = 0; d < kIndexerHeadDim; d += 16)
|
||||
store_from_float_ext<at::BFloat16>(quant_bf16.data() + d, fVec::loadu(normed.data() + d));
|
||||
|
||||
const int64_t block_idx = kv_slot / kv_cache_block_size;
|
||||
const int64_t pos_in_block = kv_slot % kv_cache_block_size;
|
||||
uint8_t* const block_base = kv_cache + block_idx * cache_row_stride;
|
||||
uint8_t* const token_fp8_ptr = block_base + pos_in_block * kIndexerTokenBytes;
|
||||
uint8_t* const token_scale_ptr =
|
||||
block_base + kv_cache_block_size * kIndexerTokenBytes + pos_in_block * kIndexerScaleBytes;
|
||||
|
||||
float absmax = 1e-4f;
|
||||
{
|
||||
__m512 max_v_reg = _mm512_set1_ps(absmax);
|
||||
for (int64_t di = 0; di < kIndexerHeadDim; di += 16) {
|
||||
__m256i bf16_v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(quant_bf16.data() + di));
|
||||
__m512 f = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_v), 16));
|
||||
__m512 av = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f), _mm512_set1_epi32(0x7FFFFFFF)));
|
||||
max_v_reg = _mm512_max_ps(max_v_reg, av);
|
||||
}
|
||||
absmax = _mm512_reduce_max_ps(max_v_reg);
|
||||
}
|
||||
const float exponent = std::ceil(std::log2(absmax / kFp8Max));
|
||||
const float scale = std::exp2(exponent);
|
||||
const float inv_scale = 1.0f / scale;
|
||||
|
||||
// AVX512 quantize+store, reusing SGLang's `cvt_fp32x16_to_fp8x16` bit-
|
||||
// manipulation conversion (this single-shared-scale 128-wide block has
|
||||
// no direct SGLang equivalent -- see file header -- so only the fp8
|
||||
// conversion primitive is ported, the amax/scale formula is unchanged
|
||||
// from vLLM's own reference).
|
||||
uint8_t* const out_fp8 = token_fp8_ptr;
|
||||
{
|
||||
const __m512 vinv = _mm512_set1_ps(inv_scale);
|
||||
const __m512 vmax = _mm512_set1_ps(kFp8Max);
|
||||
const __m512 vmin = _mm512_set1_ps(kFp8Min);
|
||||
alignas(64) float scaled_buf[kIndexerHeadDim];
|
||||
for (int64_t d = 0; d < kIndexerHeadDim; d += 16) {
|
||||
__m256i bf16_v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(quant_bf16.data() + d));
|
||||
__m512 f = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_v), 16));
|
||||
__m512 s = _mm512_mul_ps(f, vinv);
|
||||
s = _mm512_max_ps(_mm512_min_ps(s, vmax), vmin);
|
||||
_mm512_store_ps(scaled_buf + d, s);
|
||||
__m128i fp8_v = cvt_fp32x16_to_fp8x16(s);
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i*>(out_fp8 + d), fp8_v);
|
||||
}
|
||||
// Patch subnormal-magnitude lanes (see `quantize_tile_avx512`'s
|
||||
// comment above for why the vectorized conversion's flush-to-zero
|
||||
// must not be trusted here).
|
||||
for (int64_t d = 0; d < kIndexerHeadDim; ++d) {
|
||||
const float av = std::abs(scaled_buf[d]);
|
||||
if (av != 0.f && av < 0x1p-6f) {
|
||||
out_fp8[d] = at::Float8_e4m3fn(scaled_buf[d]).x;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Raw fp32 scale (not UE8M0-encoded) -- single quant block, matches
|
||||
// fp8_paged_mqa_logits_cpu's (paged_mqa_logits.cpp) read-side expectations.
|
||||
*reinterpret_cast<float*>(token_scale_ptr) = scale;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
void save_partial_states_cpu(
|
||||
at::Tensor& kv,
|
||||
at::Tensor& score,
|
||||
at::Tensor& ape,
|
||||
at::Tensor& positions,
|
||||
at::Tensor& state_cache,
|
||||
at::Tensor& slot_mapping) {
|
||||
TORCH_CHECK(
|
||||
kv.dim() == 2 && kv.scalar_type() == at::kFloat, "save_partial_states_cpu: kv must be a 2D fp32 tensor");
|
||||
TORCH_CHECK(score.sizes() == kv.sizes() && score.scalar_type() == at::kFloat,
|
||||
"save_partial_states_cpu: score must match kv's shape/dtype");
|
||||
TORCH_CHECK(ape.dim() == 2 && ape.size(1) == kv.size(1) && ape.scalar_type() == at::kFloat,
|
||||
"save_partial_states_cpu: ape must be [compress_ratio, state_width] fp32");
|
||||
TORCH_CHECK(positions.scalar_type() == at::kLong && positions.numel() == kv.size(0),
|
||||
"save_partial_states_cpu: positions must be int64 with num_tokens entries");
|
||||
TORCH_CHECK(slot_mapping.scalar_type() == at::kLong && slot_mapping.numel() == kv.size(0),
|
||||
"save_partial_states_cpu: slot_mapping must be int64 with num_tokens entries");
|
||||
TORCH_CHECK(state_cache.dim() == 3 && state_cache.size(2) == 2 * kv.size(1) &&
|
||||
state_cache.scalar_type() == at::kFloat,
|
||||
"save_partial_states_cpu: state_cache must be [num_blocks, block_size, "
|
||||
"2*state_width] fp32");
|
||||
TORCH_CHECK(state_cache.stride(2) == 1 && state_cache.stride(1) == state_cache.size(2),
|
||||
"save_partial_states_cpu: state_cache rows must be contiguous within a block");
|
||||
TORCH_CHECK(
|
||||
kv.device().is_cpu() && score.device().is_cpu() && ape.device().is_cpu() && positions.device().is_cpu() &&
|
||||
state_cache.device().is_cpu() && slot_mapping.device().is_cpu(),
|
||||
"save_partial_states_cpu: all inputs must be CPU tensors");
|
||||
|
||||
const int64_t num_tokens = kv.size(0);
|
||||
const int64_t state_width = kv.size(1);
|
||||
const int64_t compress_ratio = ape.size(0);
|
||||
const int64_t block_size = state_cache.size(1);
|
||||
|
||||
TORCH_CHECK(
|
||||
kv.stride(1) == 1 && score.stride(1) == 1 && ape.is_contiguous() && positions.is_contiguous() &&
|
||||
slot_mapping.is_contiguous(),
|
||||
"save_partial_states_cpu: kv/score must be contiguous in their last dim "
|
||||
"(row stride may exceed state_width -- e.g. a column-split view of a "
|
||||
"fused kv_score projection); ape/positions/slot_mapping must be fully "
|
||||
"contiguous");
|
||||
|
||||
save_partial_states_impl(
|
||||
state_cache.data_ptr<float>(),
|
||||
state_cache.stride(0),
|
||||
block_size,
|
||||
kv.data_ptr<float>(),
|
||||
kv.stride(0),
|
||||
score.data_ptr<float>(),
|
||||
score.stride(0),
|
||||
ape.data_ptr<float>(),
|
||||
positions.data_ptr<int64_t>(),
|
||||
slot_mapping.data_ptr<int64_t>(),
|
||||
state_width,
|
||||
compress_ratio,
|
||||
num_tokens);
|
||||
}
|
||||
|
||||
void compress_norm_rope_store_cpu(
|
||||
at::Tensor& state_cache,
|
||||
at::Tensor& gather_slots,
|
||||
at::Tensor& positions,
|
||||
at::Tensor& kv_slot_mapping,
|
||||
at::Tensor& rms_norm_weight,
|
||||
double rms_norm_eps,
|
||||
at::Tensor& cos_sin_cache,
|
||||
at::Tensor& kv_cache_2d,
|
||||
int64_t kv_cache_block_size,
|
||||
int64_t compress_ratio) {
|
||||
TORCH_CHECK(state_cache.dim() == 3 && state_cache.scalar_type() == at::kFloat,
|
||||
"compress_norm_rope_store_cpu: state_cache must be a 3D fp32 tensor");
|
||||
TORCH_CHECK(state_cache.stride(2) == 1 && state_cache.stride(1) == state_cache.size(2),
|
||||
"compress_norm_rope_store_cpu: state_cache rows must be contiguous within a block");
|
||||
TORCH_CHECK(gather_slots.dim() == 2 && gather_slots.scalar_type() == at::kLong,
|
||||
"compress_norm_rope_store_cpu: gather_slots must be a 2D int64 tensor");
|
||||
const int64_t num_tokens = gather_slots.size(0);
|
||||
TORCH_CHECK(positions.scalar_type() == at::kLong && positions.numel() == num_tokens,
|
||||
"compress_norm_rope_store_cpu: positions must be int64 with num_tokens entries");
|
||||
TORCH_CHECK(kv_slot_mapping.scalar_type() == at::kLong && kv_slot_mapping.numel() == num_tokens,
|
||||
"compress_norm_rope_store_cpu: kv_slot_mapping must be int64 with num_tokens entries");
|
||||
TORCH_CHECK(rms_norm_weight.numel() == kHeadDim,
|
||||
"compress_norm_rope_store_cpu: rms_norm_weight must have head_dim (512) entries");
|
||||
TORCH_CHECK(kv_cache_2d.dim() == 2 && kv_cache_2d.scalar_type() == at::kByte,
|
||||
"compress_norm_rope_store_cpu: kv_cache_2d must be a 2D uint8 tensor");
|
||||
TORCH_CHECK(state_cache.size(2) % 2 == 0, "compress_norm_rope_store_cpu: state_cache width must be even");
|
||||
TORCH_CHECK(
|
||||
rms_norm_weight.scalar_type() == at::kFloat && rms_norm_weight.is_contiguous() &&
|
||||
cos_sin_cache.scalar_type() == at::kFloat && cos_sin_cache.is_contiguous(),
|
||||
"compress_norm_rope_store_cpu: rms_norm_weight/cos_sin_cache must be "
|
||||
"float32 and contiguous (convert once in Python)");
|
||||
TORCH_CHECK(
|
||||
gather_slots.is_contiguous() && positions.is_contiguous() && kv_slot_mapping.is_contiguous(),
|
||||
"compress_norm_rope_store_cpu: gather_slots/positions/kv_slot_mapping must be contiguous");
|
||||
TORCH_CHECK(
|
||||
state_cache.device().is_cpu() && gather_slots.device().is_cpu() && positions.device().is_cpu() &&
|
||||
kv_slot_mapping.device().is_cpu() && rms_norm_weight.device().is_cpu() &&
|
||||
cos_sin_cache.device().is_cpu() && kv_cache_2d.device().is_cpu(),
|
||||
"compress_norm_rope_store_cpu: all inputs must be CPU tensors");
|
||||
|
||||
const int64_t window = gather_slots.size(1);
|
||||
const int64_t state_width = state_cache.size(2) / 2;
|
||||
const int64_t state_block_size = state_cache.size(1);
|
||||
|
||||
// Row strides are threaded through explicitly rather than assumed, since
|
||||
// both buffers may be pages of a larger pooled allocation (matches
|
||||
// store_cache.cpp/flash_mla.cpp's precedent).
|
||||
const int64_t state_block_stride = state_cache.stride(0);
|
||||
const int64_t cache_row_stride = kv_cache_2d.stride(0);
|
||||
|
||||
compress_norm_rope_store_impl(
|
||||
state_cache.data_ptr<float>(),
|
||||
state_block_stride,
|
||||
state_block_size,
|
||||
gather_slots.data_ptr<int64_t>(),
|
||||
positions.data_ptr<int64_t>(),
|
||||
kv_slot_mapping.data_ptr<int64_t>(),
|
||||
rms_norm_weight.data_ptr<float>(),
|
||||
static_cast<float>(rms_norm_eps),
|
||||
cos_sin_cache.data_ptr<float>(),
|
||||
kv_cache_2d.data_ptr<uint8_t>(),
|
||||
cache_row_stride,
|
||||
kv_cache_block_size,
|
||||
compress_ratio,
|
||||
window,
|
||||
state_width,
|
||||
num_tokens);
|
||||
}
|
||||
|
||||
void compress_norm_rope_store_indexer_cpu(
|
||||
at::Tensor& state_cache,
|
||||
at::Tensor& gather_slots,
|
||||
at::Tensor& positions,
|
||||
at::Tensor& kv_slot_mapping,
|
||||
at::Tensor& rms_norm_weight,
|
||||
double rms_norm_eps,
|
||||
at::Tensor& cos_sin_cache,
|
||||
at::Tensor& kv_cache_2d,
|
||||
int64_t kv_cache_block_size,
|
||||
int64_t compress_ratio) {
|
||||
TORCH_CHECK(state_cache.dim() == 3 && state_cache.scalar_type() == at::kFloat,
|
||||
"compress_norm_rope_store_indexer_cpu: state_cache must be a 3D fp32 tensor");
|
||||
TORCH_CHECK(state_cache.stride(2) == 1 && state_cache.stride(1) == state_cache.size(2),
|
||||
"compress_norm_rope_store_indexer_cpu: state_cache rows must be contiguous within a block");
|
||||
TORCH_CHECK(gather_slots.dim() == 2 && gather_slots.scalar_type() == at::kLong,
|
||||
"compress_norm_rope_store_indexer_cpu: gather_slots must be a 2D int64 tensor");
|
||||
const int64_t num_tokens = gather_slots.size(0);
|
||||
TORCH_CHECK(positions.scalar_type() == at::kLong && positions.numel() == num_tokens,
|
||||
"compress_norm_rope_store_indexer_cpu: positions must be int64 with num_tokens entries");
|
||||
TORCH_CHECK(kv_slot_mapping.scalar_type() == at::kLong && kv_slot_mapping.numel() == num_tokens,
|
||||
"compress_norm_rope_store_indexer_cpu: kv_slot_mapping must be int64 with num_tokens entries");
|
||||
TORCH_CHECK(rms_norm_weight.numel() == kIndexerHeadDim,
|
||||
"compress_norm_rope_store_indexer_cpu: rms_norm_weight must have head_dim (128) entries");
|
||||
TORCH_CHECK(kv_cache_2d.dim() == 2 && kv_cache_2d.scalar_type() == at::kByte,
|
||||
"compress_norm_rope_store_indexer_cpu: kv_cache_2d must be a 2D uint8 tensor");
|
||||
TORCH_CHECK(state_cache.size(2) % 2 == 0,
|
||||
"compress_norm_rope_store_indexer_cpu: state_cache width must be even");
|
||||
TORCH_CHECK(
|
||||
rms_norm_weight.scalar_type() == at::kFloat && rms_norm_weight.is_contiguous() &&
|
||||
cos_sin_cache.scalar_type() == at::kFloat && cos_sin_cache.is_contiguous(),
|
||||
"compress_norm_rope_store_indexer_cpu: rms_norm_weight/cos_sin_cache must "
|
||||
"be float32 and contiguous (convert once in Python)");
|
||||
TORCH_CHECK(
|
||||
gather_slots.is_contiguous() && positions.is_contiguous() && kv_slot_mapping.is_contiguous(),
|
||||
"compress_norm_rope_store_indexer_cpu: gather_slots/positions/kv_slot_mapping must be contiguous");
|
||||
TORCH_CHECK(
|
||||
state_cache.device().is_cpu() && gather_slots.device().is_cpu() && positions.device().is_cpu() &&
|
||||
kv_slot_mapping.device().is_cpu() && rms_norm_weight.device().is_cpu() &&
|
||||
cos_sin_cache.device().is_cpu() && kv_cache_2d.device().is_cpu(),
|
||||
"compress_norm_rope_store_indexer_cpu: all inputs must be CPU tensors");
|
||||
|
||||
const int64_t window = gather_slots.size(1);
|
||||
const int64_t state_width = state_cache.size(2) / 2;
|
||||
const int64_t state_block_size = state_cache.size(1);
|
||||
|
||||
const int64_t state_block_stride = state_cache.stride(0);
|
||||
const int64_t cache_row_stride = kv_cache_2d.stride(0);
|
||||
|
||||
compress_norm_rope_store_indexer_impl(
|
||||
state_cache.data_ptr<float>(),
|
||||
state_block_stride,
|
||||
state_block_size,
|
||||
gather_slots.data_ptr<int64_t>(),
|
||||
positions.data_ptr<int64_t>(),
|
||||
kv_slot_mapping.data_ptr<int64_t>(),
|
||||
rms_norm_weight.data_ptr<float>(),
|
||||
static_cast<float>(rms_norm_eps),
|
||||
cos_sin_cache.data_ptr<float>(),
|
||||
kv_cache_2d.data_ptr<uint8_t>(),
|
||||
cache_row_stride,
|
||||
kv_cache_block_size,
|
||||
compress_ratio,
|
||||
window,
|
||||
state_width,
|
||||
num_tokens);
|
||||
}
|
||||
@@ -0,0 +1,625 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/blob/main/python/sglang/kernels/aot/csrc/cpu/flash_mla.cpp
|
||||
//
|
||||
// DeepSeek-V4 sparse MQA attention kernel for the fp8_ds_mla cache layout.
|
||||
// Near-verbatim port of `sparse_mla_decode_kernel_impl` (VNNI K/V packing +
|
||||
// AMX brgemm QK^T/PV + online softmax); the `V4_FP8Sparse` byte layout
|
||||
// matches vLLM's fp8_ds_mla layout exactly (see store_cache.cpp).
|
||||
//
|
||||
// Differences from upstream:
|
||||
// - Outer shape: upstream takes a padded batch-major query `[B, S_q, H, D]`
|
||||
// with per-batch topk sets; vLLM passes flat `[num_tokens, H, D]` with
|
||||
// per-token resolved indices. The kernel's own `at::parallel_for` already
|
||||
// flattens `(batch, s_q)` internally, so this is called with
|
||||
// `batches = num_tokens, s_q = 1` -- no padding, no kernel-body changes.
|
||||
// - Two physically distinct paged caches (SWA "window" + compressor
|
||||
// "compressed") are addressed via block_table-resolved flat slot ids,
|
||||
// matching upstream's `k_main`/`k_extra` two-index-set convention.
|
||||
// - `attn_sink` uses upstream's own post-hoc LSE correction
|
||||
// (`out *= 1 / (1 + exp(sink - lse))`).
|
||||
// - bf16 query only (matching upstream's own restriction: its FP8->bf16
|
||||
// dequant intrinsics are bf16-specific); fp16 is not supported here.
|
||||
//
|
||||
// clang-format off
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <vector>
|
||||
|
||||
#include "common.h"
|
||||
#include "gemm.h"
|
||||
#include "vec.h"
|
||||
|
||||
#if !defined(CPU_CAPABILITY_AVX512)
|
||||
static_assert(false,
|
||||
"flash_mla.cpp is only ever compiled into the AVX512+AMX-flagged "
|
||||
"_C target (see cmake/cpu_extension.cmake's VLLM_EXT_SRC_SGL) and "
|
||||
"relies on that unconditionally -- CPU_CAPABILITY_AVX512 must be "
|
||||
"defined here.");
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int64_t kNopeDim = 448;
|
||||
constexpr int64_t kRopeDim = 64;
|
||||
constexpr int64_t kHeadDim = kNopeDim + kRopeDim;
|
||||
constexpr int64_t kQuantBlock = 64;
|
||||
constexpr int64_t kNumQuantBlocks = kNopeDim / kQuantBlock;
|
||||
constexpr int64_t kScaleBytesPerToken = kNumQuantBlocks + 1;
|
||||
constexpr int64_t kTokenDataBytes = kNopeDim + kRopeDim * 2;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// V4_FP8Sparse layout (SGLang's `flash_mla.cpp`) -- byte-for-byte identical
|
||||
// to vLLM's own fp8_ds_mla layout above (d=512, d_nope=448, d_rope=64,
|
||||
// tile=64, num_tiles=7):
|
||||
// per block:
|
||||
// [block_size * (d_nope + 2*d_rope) bytes ; FP8 NoPE + bf16 RoPE per token]
|
||||
// [block_size * 8 bytes ; 7 e8m0 scales + 1 pad byte/token]
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct FP8LayoutMeta {
|
||||
int64_t d;
|
||||
int64_t d_nope;
|
||||
int64_t d_rope;
|
||||
int64_t tile_size;
|
||||
int64_t num_tiles;
|
||||
};
|
||||
|
||||
constexpr FP8LayoutMeta kV4Meta = {kHeadDim, kNopeDim, kRopeDim, kQuantBlock, kNumQuantBlocks};
|
||||
|
||||
// Convert one fp8_e8m0 byte (= unsigned 8-bit exponent) to float.
|
||||
// e8m0 only stores an exponent (bias 127); value = 2^(e - 127), with 0xFF
|
||||
// reserved for NaN.
|
||||
inline float fp8_e8m0_to_float(uint8_t v) {
|
||||
if (v == 0xFF) return std::numeric_limits<float>::quiet_NaN();
|
||||
if (v == 0) return 0.f;
|
||||
union {
|
||||
uint32_t u;
|
||||
float f;
|
||||
} u;
|
||||
u.u = static_cast<uint32_t>(v) << 23;
|
||||
return u.f;
|
||||
}
|
||||
|
||||
template <typename index_t>
|
||||
inline bool is_valid_sparse_index(index_t idx, int64_t pos, int64_t topk_limit, int64_t total_tokens) {
|
||||
const int64_t v = static_cast<int64_t>(idx);
|
||||
return pos < topk_limit && v >= 0 && v < total_tokens;
|
||||
}
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
inline __attribute__((always_inline)) __m512i cvt_fp8_32_to_scaled_bf16(__m256i fp8, float scale) {
|
||||
const __m512bh bf16_ext = CVT_FP8_TO_BF16_EXT(fp8);
|
||||
const __m512 scale_vec = _mm512_mul_ps(_mm512_set1_ps(scale), _mm512_castsi512_ps(_mm512_set1_epi32(kFP8_BIAS)));
|
||||
const __m512 f0 = _mm512_mul_ps(CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_ext, 0)), scale_vec);
|
||||
const __m512 f1 = _mm512_mul_ps(CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_ext, 1)), scale_vec);
|
||||
return (__m512i)_mm512_cvtne2ps_pbh(f1, f0);
|
||||
}
|
||||
|
||||
// Load 32 contiguous logical dims (NoPE fp8-dequant-to-bf16, or raw bf16
|
||||
// RoPE) starting at `dim_offset` from one token row of the V4_FP8Sparse
|
||||
// layout.
|
||||
inline __attribute__((always_inline)) __m512i load_fp8_kvcache_32_from_row(
|
||||
const uint8_t* __restrict__ row_base, const uint8_t* __restrict__ scale_base, int dim_offset) {
|
||||
if (dim_offset < kV4Meta.d_nope) {
|
||||
const float scale = fp8_e8m0_to_float(scale_base[dim_offset / kV4Meta.tile_size]);
|
||||
return cvt_fp8_32_to_scaled_bf16(
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(row_base + dim_offset)), scale);
|
||||
}
|
||||
const auto* rope_ptr = reinterpret_cast<const at::BFloat16*>(row_base + kV4Meta.d_nope);
|
||||
return _mm512_loadu_si512(rope_ptr + dim_offset - kV4Meta.d_nope);
|
||||
}
|
||||
|
||||
inline __attribute__((always_inline)) void init_fp8_kvcache_tile_rows(
|
||||
const uint8_t* __restrict__ fp8_storage,
|
||||
const int64_t block_size,
|
||||
const int64_t storage_block_stride_bytes,
|
||||
const int64_t token_idx,
|
||||
const uint8_t*& row_base,
|
||||
const uint8_t*& scale_base) {
|
||||
constexpr int64_t nope_rope_per_token = kV4Meta.d_nope + 2 * kV4Meta.d_rope;
|
||||
constexpr int64_t scale_stride = 8;
|
||||
const int64_t block_idx = token_idx / block_size;
|
||||
const int64_t block_off = token_idx - block_idx * block_size;
|
||||
const uint8_t* block_base = fp8_storage + block_idx * storage_block_stride_bytes;
|
||||
row_base = block_base + block_off * nope_rope_per_token;
|
||||
scale_base = block_base + block_size * nope_rope_per_token + block_off * scale_stride;
|
||||
}
|
||||
|
||||
template <bool convert_v, typename scalar_t, typename index_t, typename load_vec_t>
|
||||
inline __attribute__((always_inline)) void sparse_pack_vnni_Nx32(
|
||||
scalar_t* __restrict__ dst0,
|
||||
scalar_t* __restrict__ dst1,
|
||||
const index_t* __restrict__ ind,
|
||||
const bool* __restrict__ valid_mask,
|
||||
int N,
|
||||
int dim_offset,
|
||||
int ld_dst0,
|
||||
int ld_dst1,
|
||||
const load_vec_t& load_vec) {
|
||||
__m512i vinputs[16];
|
||||
int n = 0;
|
||||
for (; n < N; ++n) {
|
||||
if (!valid_mask[n]) {
|
||||
vinputs[n] = _mm512_set1_epi32(0);
|
||||
} else {
|
||||
vinputs[n] = load_vec(n, static_cast<int64_t>(ind[n]), dim_offset);
|
||||
}
|
||||
}
|
||||
for (; n < 16; ++n) {
|
||||
vinputs[n] = _mm512_set1_epi32(0);
|
||||
}
|
||||
|
||||
if constexpr (convert_v) {
|
||||
for (int nn = 0; nn < 16; nn += 2) {
|
||||
__m512i d0, d1;
|
||||
std::tie(d0, d1) = transpose_2x32_16bit(vinputs[nn], vinputs[nn + 1]);
|
||||
_mm512_storeu_si512(dst1 + (nn >> 1) * ld_dst1 * 2, d0);
|
||||
_mm512_storeu_si512(dst1 + (nn >> 1) * ld_dst1 * 2 + 32, d1);
|
||||
}
|
||||
}
|
||||
|
||||
transpose_16x16_32bit(vinputs);
|
||||
const __mmask16 vmask = (1 << N) - 1;
|
||||
for (int k = 0; k < 16; ++k) {
|
||||
_mm512_mask_storeu_epi32(dst0 + k * ld_dst0 * 2, vmask, vinputs[k]);
|
||||
}
|
||||
}
|
||||
#endif // CPU_CAPABILITY_AVX512
|
||||
|
||||
// Gather+VNNI-pack `N` fp8-cache rows (dequantized to bf16 while packing)
|
||||
// into `dst0` (K, vnni-packed for brgemm's B operand) and `dst1` (V,
|
||||
// vnni-packed).
|
||||
template <typename scalar_t, typename index_t>
|
||||
void sparse_pack_vnni_fp8(
|
||||
scalar_t* __restrict__ dst0,
|
||||
scalar_t* __restrict__ dst1,
|
||||
const uint8_t* __restrict__ fp8_storage,
|
||||
const index_t* __restrict__ ind,
|
||||
const bool* __restrict__ valid_mask,
|
||||
int N,
|
||||
int K,
|
||||
int Kv,
|
||||
int64_t block_size,
|
||||
int64_t storage_block_stride_bytes,
|
||||
int ld_dst0,
|
||||
int ld_dst1) {
|
||||
const int NB = div_up(N, 16);
|
||||
const int KB = K / 32;
|
||||
const int KBv = std::min(Kv / 32, KB);
|
||||
for (int nb = 0; nb < NB; ++nb) {
|
||||
const uint8_t* row_base[16];
|
||||
const uint8_t* scale_base[16];
|
||||
const int nb_size = std::min(N - nb * 16, 16);
|
||||
for (int n = 0; n < nb_size; ++n) {
|
||||
if (!valid_mask[nb * 16 + n]) {
|
||||
row_base[n] = nullptr;
|
||||
scale_base[n] = nullptr;
|
||||
continue;
|
||||
}
|
||||
init_fp8_kvcache_tile_rows(
|
||||
fp8_storage, block_size, storage_block_stride_bytes, static_cast<int64_t>(ind[nb * 16 + n]), row_base[n],
|
||||
scale_base[n]);
|
||||
}
|
||||
auto load_vec = [&row_base, &scale_base](int n, int64_t idx, int dim_offset) {
|
||||
UNUSED(idx);
|
||||
return load_fp8_kvcache_32_from_row(row_base[n], scale_base[n], dim_offset);
|
||||
};
|
||||
for (int kb = 0; kb < KBv; ++kb) {
|
||||
sparse_pack_vnni_Nx32<true, scalar_t, index_t>(
|
||||
dst0 + ((kb * 32) >> 1) * ld_dst0 * 2 + nb * 16 * 2,
|
||||
dst1 + ((nb * 16) >> 1) * ld_dst1 * 2 + kb * 32 * 2,
|
||||
ind + nb * 16,
|
||||
valid_mask + nb * 16,
|
||||
nb_size,
|
||||
kb * 32,
|
||||
ld_dst0,
|
||||
ld_dst1,
|
||||
load_vec);
|
||||
}
|
||||
for (int kb = KBv; kb < KB; ++kb) {
|
||||
sparse_pack_vnni_Nx32<false, scalar_t, index_t>(
|
||||
dst0 + ((kb * 32) >> 1) * ld_dst0 * 2 + nb * 16 * 2,
|
||||
dst1 + ((nb * 16) >> 1) * ld_dst1 * 2 + kb * 32 * 2,
|
||||
ind + nb * 16,
|
||||
valid_mask + nb * 16,
|
||||
nb_size,
|
||||
kb * 32,
|
||||
ld_dst0,
|
||||
ld_dst1,
|
||||
load_vec);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void fmla_fill_stub(scalar_t* __restrict__ out, float val, int64_t size) {
|
||||
using Vec = at::vec::Vectorized<scalar_t>;
|
||||
constexpr int kVecSize = Vec::size();
|
||||
const Vec data_vec = Vec(static_cast<scalar_t>(val));
|
||||
int64_t d = 0;
|
||||
#pragma GCC unroll 4
|
||||
for (; d <= size - kVecSize; d += kVecSize) {
|
||||
data_vec.store(out + d);
|
||||
}
|
||||
if (size - d > 0) {
|
||||
data_vec.store(out + d, size - d);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int BLOCK_N>
|
||||
inline void fmla_copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input) {
|
||||
static_assert(BLOCK_N % 32 == 0);
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int COLS = BLOCK_N / 16;
|
||||
auto store = [&](auto i) {
|
||||
constexpr int col = i % COLS;
|
||||
if constexpr (col % 2 == 0) {
|
||||
fVec a0 = fVec::loadu(input + col * 16);
|
||||
fVec a1 = fVec::loadu(input + col * 16 + 16);
|
||||
bVec out_bvec = convert_from_float_ext<scalar_t>(a0, a1);
|
||||
out_bvec.store(out + col * 16);
|
||||
}
|
||||
};
|
||||
Unroll<COLS>{}(store);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void fmla_finalize_out(scalar_t* __restrict__ out, const float* __restrict__ acc, float inv_s, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
const fVec s_fvec = fVec(inv_s);
|
||||
int64_t d = 0;
|
||||
#pragma GCC unroll 4
|
||||
for (; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec a0 = fVec::loadu(acc + d) * s_fvec;
|
||||
fVec a1 = fVec::loadu(acc + d + fVec::size()) * s_fvec;
|
||||
bVec out_bvec = convert_from_float_ext<scalar_t>(a0, a1);
|
||||
out_bvec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(acc[d] * inv_s);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// query : [num_tokens, H_q, D_qk] bf16 (SGLang's `[B, S_q, H_q, D]`
|
||||
// with S_q folded away -- vLLM's flat per-token convention)
|
||||
// indices : [num_tokens, topk_main] int64 (flat, block-table-resolved
|
||||
// slot ids, -1 = invalid; SGLang's "main"/window index set)
|
||||
// extra_* : same, for the optional "extra"/compressed index set
|
||||
// attn_sink: [H_q] float32
|
||||
// output : [num_tokens, H_q, D_v] bf16
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
template <typename scalar_t, typename index_t, int64_t BLOCK_N>
|
||||
void sparse_mla_decode_kernel_impl(
|
||||
scalar_t* __restrict__ output,
|
||||
float* __restrict__ lse_out,
|
||||
const scalar_t* __restrict__ query,
|
||||
const uint8_t* __restrict__ k_main_fp8,
|
||||
const uint8_t* __restrict__ k_extra_fp8,
|
||||
const index_t* __restrict__ indices,
|
||||
const index_t* __restrict__ extra_indices,
|
||||
const float* __restrict__ attn_sink,
|
||||
scalar_t* __restrict__ buffer,
|
||||
int64_t batches,
|
||||
int64_t s_q,
|
||||
int64_t num_heads,
|
||||
int64_t head_size,
|
||||
int64_t head_size_v,
|
||||
int64_t topk_main,
|
||||
int64_t topk_extra,
|
||||
int64_t total_tokens_main,
|
||||
int64_t total_tokens_extra,
|
||||
int64_t q_strideB,
|
||||
int64_t q_strideH,
|
||||
int64_t k_main_block_size,
|
||||
int64_t k_extra_block_size,
|
||||
int64_t k_main_block_stride_bytes,
|
||||
int64_t k_extra_block_stride_bytes,
|
||||
int64_t idx_strideB,
|
||||
int64_t extra_idx_strideB,
|
||||
float scaling,
|
||||
int64_t buffer_size_per_thread) {
|
||||
using Vec = at::vec::Vectorized<float>;
|
||||
|
||||
// partition heads
|
||||
constexpr int64_t kBLOCK_H_MAX = 16;
|
||||
const int64_t BLOCK_H = (batches * s_q) >= 16 ? kBLOCK_H_MAX : 8;
|
||||
const int64_t num_h_blocks = div_up(num_heads, BLOCK_H);
|
||||
|
||||
// parallel on [B, S_q, head_block] -- with s_q == 1 (vLLM's flat call
|
||||
// convention) this collapses to a plain [num_tokens, head_block] loop,
|
||||
// one flat iteration per token with no padding to skip.
|
||||
at::parallel_for(0, batches * s_q * num_h_blocks, 0, [&](int64_t begin, int64_t end) {
|
||||
int64_t bs{0}, sq{0}, hb{0};
|
||||
data_index_init(begin, bs, batches, sq, s_q, hb, num_h_blocks);
|
||||
|
||||
int tid = at::get_thread_num();
|
||||
scalar_t* __restrict__ Btmp0 = buffer + tid * buffer_size_per_thread; // K packed
|
||||
scalar_t* __restrict__ Btmp1 = Btmp0 + BLOCK_N * head_size; // V packed
|
||||
// f32 V accumulator follows the bf16 packing region (reinterpret cast).
|
||||
float* __restrict__ v_acc_local = reinterpret_cast<float*>(Btmp1 + BLOCK_N * head_size_v);
|
||||
fmla_fill_stub(Btmp1, 0.f, BLOCK_N * head_size_v); // initialize V padding
|
||||
|
||||
alignas(64) float s_i[kBLOCK_H_MAX * BLOCK_N];
|
||||
float* __restrict__ s_delta = s_i;
|
||||
alignas(64) scalar_t s_delta2[kBLOCK_H_MAX * BLOCK_N];
|
||||
|
||||
alignas(64) float s_prime[kBLOCK_H_MAX];
|
||||
alignas(64) float m_prime[kBLOCK_H_MAX];
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
const int64_t h_start = hb * BLOCK_H;
|
||||
const int64_t h_end = std::min(h_start + BLOCK_H, num_heads);
|
||||
const int64_t h_size = h_end - h_start;
|
||||
|
||||
const scalar_t* __restrict__ q_ptr = query + bs * q_strideB + h_start * q_strideH;
|
||||
const index_t* __restrict__ idx_ptr = indices + bs * idx_strideB;
|
||||
const index_t* __restrict__ extra_idx_ptr = extra_indices == nullptr ? nullptr : extra_indices + bs * extra_idx_strideB;
|
||||
|
||||
fmla_fill_stub(s_prime, 0.f, BLOCK_H);
|
||||
fmla_fill_stub(m_prime, -std::numeric_limits<float>::infinity(), BLOCK_H);
|
||||
for (int64_t h = 0; h < h_size; ++h) {
|
||||
fmla_fill_stub(v_acc_local + h * head_size_v, 0.f, head_size_v);
|
||||
}
|
||||
|
||||
auto process_cache = [&](const uint8_t* __restrict__ fp8_ptr,
|
||||
const index_t* __restrict__ cur_idx_ptr,
|
||||
int64_t topk_count,
|
||||
int64_t total_tokens,
|
||||
int64_t fp8_block_size,
|
||||
int64_t fp8_block_stride_bytes) {
|
||||
if (fp8_ptr == nullptr || cur_idx_ptr == nullptr || topk_count == 0) {
|
||||
return;
|
||||
}
|
||||
const int64_t topk_limit = topk_count;
|
||||
|
||||
for (int64_t n = 0; n < topk_limit; n += BLOCK_N) {
|
||||
int64_t n_size = std::min<int64_t>(BLOCK_N, topk_limit - n);
|
||||
const int64_t padded_n_size = div_up(int(n_size), TILE_K) * TILE_K;
|
||||
bool valid_mask[BLOCK_N];
|
||||
bool has_valid = false;
|
||||
for (int64_t k = 0; k < n_size; ++k) {
|
||||
const bool valid = is_valid_sparse_index(cur_idx_ptr[n + k], n + k, topk_limit, total_tokens);
|
||||
valid_mask[k] = valid;
|
||||
has_valid |= valid;
|
||||
}
|
||||
if (!has_valid) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Pack K (BLOCK_N rows via gather) into Btmp0 (key, vnni) and Btmp1
|
||||
// (value, vnni), dequantizing FP8 cache entries while packing.
|
||||
sparse_pack_vnni_fp8<scalar_t, index_t>(
|
||||
Btmp0, Btmp1, fp8_ptr, cur_idx_ptr + n, valid_mask, static_cast<int>(n_size), static_cast<int>(head_size),
|
||||
static_cast<int>(head_size_v), fp8_block_size, fp8_block_stride_bytes, static_cast<int>(BLOCK_N),
|
||||
static_cast<int>(head_size_v));
|
||||
|
||||
// Q @ K
|
||||
at::native::cpublas::brgemm(
|
||||
/* M */ h_size,
|
||||
/* N */ n_size,
|
||||
/* K */ head_size,
|
||||
/* lda */ q_strideH,
|
||||
/* ldb */ BLOCK_N,
|
||||
/* ldc */ BLOCK_N,
|
||||
/* add_C */ false,
|
||||
/* A */ q_ptr,
|
||||
/* B */ Btmp0,
|
||||
/* C */ s_i);
|
||||
|
||||
const Vec scale_vec = Vec(scaling);
|
||||
for (int64_t h = 0; h < h_size; ++h) {
|
||||
float* row = s_i + h * BLOCK_N;
|
||||
at::vec::map<float>([scale_vec](Vec x) { return x * scale_vec; }, row, row, n_size);
|
||||
|
||||
for (int64_t k = 0; k < n_size; ++k) {
|
||||
if (!valid_mask[k]) {
|
||||
row[k] = -std::numeric_limits<float>::infinity();
|
||||
}
|
||||
}
|
||||
|
||||
// online softmax update
|
||||
float m_i = at::vec::reduce_all<float>([](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, row, n_size);
|
||||
m_i = std::max(m_i, m_prime[h]);
|
||||
|
||||
if (!std::isfinite(m_i)) {
|
||||
fmla_fill_stub(s_delta + h * BLOCK_N, 0.f, padded_n_size);
|
||||
fmla_copy_stub<scalar_t, BLOCK_N>(s_delta2 + h * BLOCK_N, s_delta + h * BLOCK_N);
|
||||
continue;
|
||||
}
|
||||
|
||||
const float m_delta = std::exp(m_prime[h] - m_i);
|
||||
at::vec::map<float>([m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + h * BLOCK_N, row, n_size);
|
||||
|
||||
s_prime[h] *= m_delta;
|
||||
s_prime[h] += at::vec::reduce_all<float>([](Vec& x, Vec& y) { return x + y; }, s_delta + h * BLOCK_N, n_size);
|
||||
|
||||
m_prime[h] = m_i;
|
||||
|
||||
float scale_m = m_delta;
|
||||
at::vec::map<float>(
|
||||
[scale_m](Vec x) { return x * Vec(scale_m); }, v_acc_local + h * head_size_v, v_acc_local + h * head_size_v,
|
||||
head_size_v);
|
||||
|
||||
fmla_fill_stub(s_delta + h * BLOCK_N + n_size, 0.f, padded_n_size - n_size);
|
||||
fmla_copy_stub<scalar_t, BLOCK_N>(s_delta2 + h * BLOCK_N, s_delta + h * BLOCK_N);
|
||||
}
|
||||
|
||||
// V' <- s_delta @ V + V' (accumulate into v_acc_local at f32)
|
||||
at::native::cpublas::brgemm(
|
||||
/* M */ h_size,
|
||||
/* N */ head_size_v,
|
||||
/* K */ padded_n_size,
|
||||
/* lda */ BLOCK_N,
|
||||
/* ldb */ head_size_v,
|
||||
/* ldc */ head_size_v,
|
||||
/* add_C */ true,
|
||||
/* A */ s_delta2,
|
||||
/* B */ Btmp1,
|
||||
/* C */ v_acc_local);
|
||||
}
|
||||
};
|
||||
|
||||
process_cache(k_main_fp8, idx_ptr, topk_main, total_tokens_main, k_main_block_size, k_main_block_stride_bytes);
|
||||
process_cache(
|
||||
k_extra_fp8, extra_idx_ptr, topk_extra, total_tokens_extra, k_extra_block_size, k_extra_block_stride_bytes);
|
||||
|
||||
// Apply attention sink correction directly on the output and lse.
|
||||
// out *= exp(lse_no_sink) / (exp(lse_no_sink) + exp(attn_sink))
|
||||
// = 1 / (1 + exp(attn_sink - lse_no_sink))
|
||||
// (Algebraically equivalent to treating attn_sink as a virtual,
|
||||
// zero-value extra key inside the softmax.)
|
||||
for (int64_t h = 0; h < h_size; ++h) {
|
||||
const int64_t hh = h_start + h;
|
||||
const bool lonely = !std::isfinite(m_prime[h]) || s_prime[h] == 0.f;
|
||||
float lse_val = lonely ? std::numeric_limits<float>::infinity() : (m_prime[h] + std::log(s_prime[h]));
|
||||
float inv_s = lonely ? 0.f : (1.f / s_prime[h]);
|
||||
|
||||
if (!lonely && attn_sink != nullptr) {
|
||||
const float sink = attn_sink[hh];
|
||||
const float corr = 1.f / (1.f + std::exp(sink - lse_val));
|
||||
inv_s *= corr;
|
||||
}
|
||||
|
||||
scalar_t* out_row = output + bs * (s_q * num_heads * head_size_v) + sq * (num_heads * head_size_v) + hh * head_size_v;
|
||||
if (lonely) {
|
||||
fmla_fill_stub(out_row, 0.f, head_size_v);
|
||||
} else {
|
||||
fmla_finalize_out<scalar_t>(out_row, v_acc_local + h * head_size_v, inv_s, head_size_v);
|
||||
}
|
||||
|
||||
lse_out[bs * num_heads * s_q + hh * s_q + sq] = lse_val;
|
||||
}
|
||||
|
||||
data_index_step(bs, batches, sq, s_q, hb, num_h_blocks);
|
||||
}
|
||||
at::native::cpublas::brgemm_release();
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
void flash_mla_with_kvcache_cpu(
|
||||
at::Tensor& out,
|
||||
at::Tensor& q,
|
||||
at::Tensor& window_cache_2d,
|
||||
at::Tensor& window_slots,
|
||||
int64_t window_block_size,
|
||||
at::Tensor& compressed_cache_2d,
|
||||
at::Tensor& compressed_slots,
|
||||
int64_t compressed_block_size,
|
||||
at::Tensor& attn_sink,
|
||||
double scale) {
|
||||
TORCH_CHECK(
|
||||
q.dim() == 3 && q.size(2) == kHeadDim,
|
||||
"flash_mla_with_kvcache_cpu: q must be [num_tokens, num_heads, 512]");
|
||||
TORCH_CHECK(
|
||||
out.dim() == 3 && out.size(0) == q.size(0) && out.size(1) == q.size(1) && out.size(2) == kHeadDim,
|
||||
"flash_mla_with_kvcache_cpu: out must be [num_tokens, num_heads, 512]");
|
||||
TORCH_CHECK(
|
||||
out.scalar_type() == at::kBFloat16 && out.is_contiguous(),
|
||||
"flash_mla_with_kvcache_cpu: out must be bf16 and contiguous "
|
||||
"(pre-allocate once in Python -- see DeepseekV4CPUAttention.forward_mqa)");
|
||||
TORCH_CHECK(out.device().is_cpu(), "flash_mla_with_kvcache_cpu: out must be a CPU tensor");
|
||||
TORCH_CHECK(
|
||||
q.scalar_type() == at::kBFloat16,
|
||||
"flash_mla_with_kvcache_cpu: q must be bf16 (the AMX/VNNI kernel's "
|
||||
"FP8->bf16 dequant intrinsics are bf16-specific)");
|
||||
TORCH_CHECK(
|
||||
window_slots.dim() == 2 && window_slots.size(0) == q.size(0),
|
||||
"flash_mla_with_kvcache_cpu: window_slots must be [num_tokens, num_window]");
|
||||
TORCH_CHECK(
|
||||
compressed_slots.dim() == 2 && compressed_slots.size(0) == q.size(0),
|
||||
"flash_mla_with_kvcache_cpu: compressed_slots must be [num_tokens, num_compressed]");
|
||||
TORCH_CHECK(
|
||||
window_slots.scalar_type() == at::kLong && compressed_slots.scalar_type() == at::kLong,
|
||||
"flash_mla_with_kvcache_cpu: slot tensors must be int64");
|
||||
TORCH_CHECK(
|
||||
attn_sink.numel() == q.size(1), "flash_mla_with_kvcache_cpu: attn_sink must have num_heads entries");
|
||||
TORCH_CHECK(
|
||||
attn_sink.scalar_type() == at::kFloat && attn_sink.is_contiguous(),
|
||||
"flash_mla_with_kvcache_cpu: attn_sink must be float32 and contiguous "
|
||||
"(convert once in Python -- see DeepseekV4CPUAttention.forward_mqa)");
|
||||
TORCH_CHECK(
|
||||
window_slots.is_contiguous() && compressed_slots.is_contiguous(),
|
||||
"flash_mla_with_kvcache_cpu: window_slots/compressed_slots must be contiguous");
|
||||
TORCH_CHECK(
|
||||
q.device().is_cpu() && window_cache_2d.device().is_cpu() && window_slots.device().is_cpu() &&
|
||||
compressed_cache_2d.device().is_cpu() && compressed_slots.device().is_cpu() &&
|
||||
attn_sink.device().is_cpu(),
|
||||
"flash_mla_with_kvcache_cpu: all inputs must be CPU tensors");
|
||||
|
||||
const int64_t num_tokens = q.size(0);
|
||||
const int64_t num_heads = q.size(1);
|
||||
const int64_t num_window = window_slots.size(1);
|
||||
const int64_t num_compressed = compressed_slots.size(1);
|
||||
TORCH_CHECK(num_window > 0, "flash_mla_with_kvcache_cpu: window_slots must have at least one column");
|
||||
|
||||
// Row stride is threaded through explicitly rather than assumed to equal
|
||||
// block_size * (token_data_bytes + scale_bytes_per_token): a paged/pooled
|
||||
// cache buffer's physical row stride is not guaranteed to match the
|
||||
// logical per-block byte count (matches store_cache.cpp's precedent).
|
||||
const int64_t window_cache_row_stride = window_cache_2d.stride(0);
|
||||
const int64_t compressed_cache_row_stride = num_compressed > 0 ? compressed_cache_2d.stride(0) : 0;
|
||||
|
||||
// AMX/VNNI path: flatten (batch, s_q) -> batches = num_tokens, s_q = 1.
|
||||
CHECK_LAST_DIM_CONTIGUOUS_INPUT(q);
|
||||
constexpr int64_t BLOCK_N = 128; // multiple of 32 for AMX brgemm
|
||||
constexpr int64_t kBLOCK_H_MAX = 16;
|
||||
static_assert(kHeadDim % 32 == 0, "head_dim must be a multiple of 32");
|
||||
|
||||
const int num_threads = at::get_num_threads();
|
||||
// Layout per thread (in bf16 elements):
|
||||
// [Btmp0 : BLOCK_N * D] [Btmp1 : BLOCK_N * D] [v_acc_local : kBLOCK_H_MAX * D floats]
|
||||
// f32 takes 2 bf16 elements -> multiply by 2.
|
||||
const int64_t buffer_size_per_thread = BLOCK_N * kHeadDim + BLOCK_N * kHeadDim + 2 * kBLOCK_H_MAX * kHeadDim;
|
||||
auto buffer = at::empty({num_threads, buffer_size_per_thread}, q.options());
|
||||
// lse is an internal scratch (attn_sink correction needs it per-token/head)
|
||||
// -- not surfaced to the Python call site, which only consumes `out`.
|
||||
auto lse = at::empty({num_tokens, num_heads}, q.options().dtype(at::kFloat));
|
||||
|
||||
const int64_t q_strideB = q.stride(0);
|
||||
const int64_t q_strideH = q.stride(1);
|
||||
const int64_t idx_strideB = window_slots.stride(0);
|
||||
const int64_t extra_idx_strideB = num_compressed > 0 ? compressed_slots.stride(0) : 0;
|
||||
|
||||
const int64_t total_tokens_main = window_cache_2d.size(0) * window_block_size;
|
||||
const int64_t total_tokens_extra = num_compressed > 0 ? compressed_cache_2d.size(0) * compressed_block_size : 0;
|
||||
|
||||
sparse_mla_decode_kernel_impl<at::BFloat16, int64_t, BLOCK_N>(
|
||||
out.data_ptr<at::BFloat16>(),
|
||||
lse.data_ptr<float>(),
|
||||
q.data_ptr<at::BFloat16>(),
|
||||
window_cache_2d.data_ptr<uint8_t>(),
|
||||
num_compressed > 0 ? compressed_cache_2d.data_ptr<uint8_t>() : nullptr,
|
||||
window_slots.data_ptr<int64_t>(),
|
||||
num_compressed > 0 ? compressed_slots.data_ptr<int64_t>() : nullptr,
|
||||
attn_sink.data_ptr<float>(),
|
||||
buffer.data_ptr<at::BFloat16>(),
|
||||
/* batches */ num_tokens,
|
||||
/* s_q */ 1,
|
||||
num_heads,
|
||||
/* head_size */ kHeadDim,
|
||||
/* head_size_v */ kHeadDim,
|
||||
/* topk_main */ num_window,
|
||||
/* topk_extra */ num_compressed,
|
||||
total_tokens_main,
|
||||
total_tokens_extra,
|
||||
q_strideB,
|
||||
q_strideH,
|
||||
window_block_size,
|
||||
num_compressed > 0 ? compressed_block_size : 1,
|
||||
window_cache_row_stride,
|
||||
compressed_cache_row_stride,
|
||||
idx_strideB,
|
||||
extra_idx_strideB,
|
||||
static_cast<float>(scale),
|
||||
buffer_size_per_thread);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
// vLLM-native: DeepSeek-V4 sparse indexer Q-side RoPE + FP8 quant kernel.
|
||||
// Ports the math of vLLM's own GPU/Triton reference
|
||||
// (`_fused_indexer_q_rope_quant_kernel`, common/ops/fused_indexer_q.py) --
|
||||
// not a port of SGLang's own indexer-Q kernel, since that applies RoPE plus
|
||||
// a 128-point Hadamard rotation the real DeepSeek-V4-Flash checkpoint
|
||||
// doesn't use (verified: it applies RoPE only, no Hadamard step).
|
||||
//
|
||||
// Per (token, head): GPT-J interleaved RoPE on the last `rot_dim` dims;
|
||||
// leading NoPE dims pass through unrotated. RoPE-rotated values are bf16-
|
||||
// round-tripped before the absmax reduction (matching the K-side compressor
|
||||
// kernel), NoPE values are not. One UE8M0 scale per (token, head) covers the
|
||||
// full head_dim; since the FP8 Q tensor has no companion scale tensor, the
|
||||
// scale is folded into `index_weights_out` instead:
|
||||
// weights_out = weights * q_scale * softmax_scale * head_scale
|
||||
// MXFP4 (`use_fp4=True`) has per-block scales that can't be folded this way
|
||||
// and stays on triton-cpu -- out of scope here.
|
||||
//
|
||||
// AVX512 path: NoPE copy+absmax and the FP8 scale/clamp use plain 16-wide
|
||||
// vector ops (final FP8 cast stays scalar, matching SGLang's own
|
||||
// `act_quant_cpu_impl`). The RoPE rotation vectorizes the interleaved
|
||||
// (x_even, x_odd) pairs via in-register permutes (`_mm512_permute_ps` /
|
||||
// `_mm512_permutexvar_ps`), the same technique compressor.cpp's
|
||||
// `apply_gptj_rope_inplace` and store_cache.cpp's
|
||||
// `inverse_gptj_rope_o_proj_cpu` use, since gather/scatter is comparatively
|
||||
// expensive on AVX512.
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
|
||||
#if !defined(CPU_CAPABILITY_AVX512)
|
||||
static_assert(
|
||||
false,
|
||||
"indexer.cpp is only ever compiled into the AVX512+AMX-flagged "
|
||||
"_C target (see cmake/cpu_extension.cmake's VLLM_EXT_SRC_SGL) and "
|
||||
"relies on that unconditionally -- CPU_CAPABILITY_AVX512 must be "
|
||||
"defined here.");
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr float kFp8Max = 448.0f;
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
|
||||
// fp32 -> bf16 -> fp32 round trip via the AVX512-BF16 conversion
|
||||
// instruction, numerically matching `static_cast<float>(at::BFloat16(x))`
|
||||
// (round-to-nearest-even).
|
||||
inline __m512 bf16_round_trip_avx512(__m512 x) {
|
||||
return CVT_BF16_TO_FP32((__m256i)(_mm512_cvtneps_pbh(x)));
|
||||
}
|
||||
|
||||
inline __m512 abs_ps(__m512 x) {
|
||||
return _mm512_andnot_ps(_mm512_set1_ps(-0.0f), x);
|
||||
}
|
||||
|
||||
#endif // CPU_CAPABILITY_AVX512
|
||||
|
||||
void fused_indexer_q_rope_quant_impl(
|
||||
const int64_t* __restrict__ positions, // [num_tokens]
|
||||
const float* __restrict__ index_q, // [num_tokens, num_heads, head_dim]
|
||||
int64_t index_q_stride0, int64_t index_q_stride1,
|
||||
const float* __restrict__ cos_sin_cache, // [max_pos, rot_dim]
|
||||
int64_t cos_sin_stride, int64_t half_rot_dim,
|
||||
at::Float8_e4m3fn* __restrict__ index_q_fp8, // [num_tokens, num_heads,
|
||||
// head_dim]
|
||||
int64_t index_q_fp8_stride0, int64_t index_q_fp8_stride1, int64_t head_dim,
|
||||
const float* __restrict__ index_weights, // [num_tokens, num_heads]
|
||||
int64_t index_weights_stride, float softmax_scale, float head_scale,
|
||||
float* __restrict__ index_weights_out, // [num_tokens, num_heads]
|
||||
int64_t index_weights_out_stride, int64_t num_tokens, int64_t num_heads) {
|
||||
const int64_t rot_dim = 2 * half_rot_dim;
|
||||
const int64_t nope_dim = head_dim - rot_dim;
|
||||
|
||||
at::parallel_for(
|
||||
0, num_tokens * num_heads, 0, [&](int64_t begin, int64_t end) {
|
||||
std::vector<float> buf(head_dim);
|
||||
for (int64_t idx = begin; idx < end; ++idx) {
|
||||
const int64_t t = idx / num_heads;
|
||||
const int64_t h = idx % num_heads;
|
||||
|
||||
const int64_t pos = positions[t];
|
||||
const float* const cos = cos_sin_cache + pos * cos_sin_stride;
|
||||
const float* const sin = cos + half_rot_dim;
|
||||
|
||||
const float* const q =
|
||||
index_q + t * index_q_stride0 + h * index_q_stride1;
|
||||
|
||||
float amax = 1e-4f;
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
{
|
||||
__m512 vamax = _mm512_set1_ps(amax);
|
||||
for (int64_t d = 0; d < nope_dim; d += 16) {
|
||||
__m512 v = _mm512_loadu_ps(q + d);
|
||||
_mm512_storeu_ps(&buf[d], v);
|
||||
vamax = _mm512_max_ps(vamax, abs_ps(v));
|
||||
}
|
||||
amax = _mm512_reduce_max_ps(vamax);
|
||||
}
|
||||
#endif
|
||||
|
||||
const float* const q_rot = q + nope_dim;
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
{
|
||||
// Permute-based (no gather/scatter), same technique as
|
||||
// compressor.cpp's apply_gptj_rope_inplace / store_cache.cpp's
|
||||
// inverse_gptj_rope_o_proj_cpu: `_mm512_permute_ps` swaps each
|
||||
// pair's (even, odd) lanes in one shuffle, and cos/sin -- loaded as
|
||||
// 8 contiguous values -- are duplicated pairwise via
|
||||
// `_mm512_permutexvar_ps` to align with that layout. Two batches of
|
||||
// 8 pairs per iteration (16 total, matching the old gather step).
|
||||
// Unlike the RoPE kernels above, the result here stays in its
|
||||
// original interleaved layout, so no un-permute/scatter is needed
|
||||
// before the bf16 round-trip and store.
|
||||
const __m512i dup_idx = _mm512_set_epi32(7, 7, 6, 6, 5, 5, 4, 4, 3,
|
||||
3, 2, 2, 1, 1, 0, 0);
|
||||
// Forward GPT-J rotation negates the sin broadcast on the even
|
||||
// (real) lane of each pair; the odd (imag) lane keeps sin's sign.
|
||||
const __m512 sign_mask = _mm512_castsi512_ps(_mm512_set_epi32(
|
||||
0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0,
|
||||
(int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0,
|
||||
(int)0x80000000, 0, (int)0x80000000));
|
||||
__m512 vamax_rope = _mm512_setzero_ps();
|
||||
for (int64_t i = 0; i < half_rot_dim; i += 16) {
|
||||
const float* const base = q_rot + 2 * i;
|
||||
float* const dst = buf.data() + nope_dim + 2 * i;
|
||||
|
||||
__m512 xv0 = _mm512_loadu_ps(base);
|
||||
__m512 cos_b0 = _mm512_permutexvar_ps(
|
||||
dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(cos + i)));
|
||||
__m512 sin_b0 = _mm512_permutexvar_ps(
|
||||
dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(sin + i)));
|
||||
__m512 out0 = _mm512_fmadd_ps(
|
||||
xv0, cos_b0,
|
||||
_mm512_mul_ps(_mm512_permute_ps(xv0, 0xB1),
|
||||
_mm512_xor_ps(sin_b0, sign_mask)));
|
||||
// Match reference numerics: fp32 -> bf16 -> fp32 before the
|
||||
// absmax.
|
||||
out0 = bf16_round_trip_avx512(out0);
|
||||
_mm512_storeu_ps(dst, out0);
|
||||
vamax_rope = _mm512_max_ps(vamax_rope, abs_ps(out0));
|
||||
|
||||
__m512 xv1 = _mm512_loadu_ps(base + 16);
|
||||
__m512 cos_b1 = _mm512_permutexvar_ps(
|
||||
dup_idx,
|
||||
_mm512_castps256_ps512(_mm256_loadu_ps(cos + i + 8)));
|
||||
__m512 sin_b1 = _mm512_permutexvar_ps(
|
||||
dup_idx,
|
||||
_mm512_castps256_ps512(_mm256_loadu_ps(sin + i + 8)));
|
||||
__m512 out1 = _mm512_fmadd_ps(
|
||||
xv1, cos_b1,
|
||||
_mm512_mul_ps(_mm512_permute_ps(xv1, 0xB1),
|
||||
_mm512_xor_ps(sin_b1, sign_mask)));
|
||||
out1 = bf16_round_trip_avx512(out1);
|
||||
_mm512_storeu_ps(dst + 16, out1);
|
||||
vamax_rope = _mm512_max_ps(vamax_rope, abs_ps(out1));
|
||||
}
|
||||
amax = std::max(amax, _mm512_reduce_max_ps(vamax_rope));
|
||||
}
|
||||
#endif
|
||||
|
||||
const float exponent = std::ceil(std::log2(amax / kFp8Max));
|
||||
const float scale = std::exp2(exponent);
|
||||
const float inv_scale = 1.0f / scale;
|
||||
|
||||
at::Float8_e4m3fn* const out =
|
||||
index_q_fp8 + t * index_q_fp8_stride0 + h * index_q_fp8_stride1;
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
{
|
||||
const __m512 vinv = _mm512_set1_ps(inv_scale);
|
||||
const __m512 vmax = _mm512_set1_ps(kFp8Max);
|
||||
const __m512 vmin = _mm512_set1_ps(-kFp8Max);
|
||||
alignas(64) float scaled_buf[16];
|
||||
for (int64_t d2 = 0; d2 < head_dim; d2 += 16) {
|
||||
__m512 v = _mm512_loadu_ps(&buf[d2]);
|
||||
v = _mm512_min_ps(_mm512_max_ps(_mm512_mul_ps(v, vinv), vmin),
|
||||
vmax);
|
||||
_mm512_store_ps(scaled_buf, v);
|
||||
for (int64_t j = 0; j < 16; ++j)
|
||||
out[d2 + j] = at::Float8_e4m3fn(scaled_buf[j]);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
float w = index_weights[t * index_weights_stride + h];
|
||||
w *= scale;
|
||||
w *= softmax_scale;
|
||||
w *= head_scale;
|
||||
index_weights_out[t * index_weights_out_stride + h] = w;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
void fused_indexer_q_rope_quant_cpu(at::Tensor& positions, at::Tensor& index_q,
|
||||
at::Tensor& index_q_cos_sin_cache,
|
||||
at::Tensor& index_q_fp8,
|
||||
at::Tensor& index_weights,
|
||||
double index_weights_softmax_scale,
|
||||
double index_weights_head_scale,
|
||||
at::Tensor& index_weights_out) {
|
||||
TORCH_CHECK(
|
||||
positions.dim() == 1 && positions.scalar_type() == at::kLong,
|
||||
"fused_indexer_q_rope_quant_cpu: positions must be a 1D int64 tensor");
|
||||
TORCH_CHECK(positions.is_contiguous(),
|
||||
"fused_indexer_q_rope_quant_cpu: positions must be contiguous");
|
||||
TORCH_CHECK(index_q.dim() == 3,
|
||||
"fused_indexer_q_rope_quant_cpu: index_q must be 3D");
|
||||
TORCH_CHECK(index_q.scalar_type() == at::kFloat,
|
||||
"fused_indexer_q_rope_quant_cpu: index_q must be float32 "
|
||||
"(convert once in Python)");
|
||||
TORCH_CHECK(index_q.stride(2) == 1,
|
||||
"fused_indexer_q_rope_quant_cpu: index_q must be contiguous in "
|
||||
"the head_dim axis");
|
||||
TORCH_CHECK(
|
||||
index_q_cos_sin_cache.dim() == 2 && index_q_cos_sin_cache.stride(1) == 1,
|
||||
"fused_indexer_q_rope_quant_cpu: index_q_cos_sin_cache must be a "
|
||||
"contiguous 2D tensor");
|
||||
TORCH_CHECK(
|
||||
index_q_cos_sin_cache.scalar_type() == at::kFloat,
|
||||
"fused_indexer_q_rope_quant_cpu: index_q_cos_sin_cache must be float32 "
|
||||
"(convert once in Python)");
|
||||
TORCH_CHECK(index_q_fp8.sizes() == index_q.sizes() &&
|
||||
index_q_fp8.scalar_type() == at::kFloat8_e4m3fn &&
|
||||
index_q_fp8.stride(2) == 1,
|
||||
"fused_indexer_q_rope_quant_cpu: index_q_fp8 must match "
|
||||
"index_q's shape, be "
|
||||
"float8_e4m3fn, and contiguous in the head_dim axis");
|
||||
TORCH_CHECK(index_weights.dim() == 2 && index_weights.stride(1) == 1,
|
||||
"fused_indexer_q_rope_quant_cpu: index_weights must be a "
|
||||
"contiguous-in-head 2D tensor");
|
||||
TORCH_CHECK(index_weights.scalar_type() == at::kFloat,
|
||||
"fused_indexer_q_rope_quant_cpu: index_weights must be float32 "
|
||||
"(convert once in Python)");
|
||||
TORCH_CHECK(index_weights_out.sizes() == index_weights.sizes() &&
|
||||
index_weights_out.scalar_type() == at::kFloat &&
|
||||
index_weights_out.stride(1) == 1,
|
||||
"fused_indexer_q_rope_quant_cpu: index_weights_out must match "
|
||||
"index_weights' shape, "
|
||||
"be float32, and contiguous in the head axis");
|
||||
TORCH_CHECK(positions.device().is_cpu() && index_q.device().is_cpu() &&
|
||||
index_q_cos_sin_cache.device().is_cpu() &&
|
||||
index_q_fp8.device().is_cpu() &&
|
||||
index_weights.device().is_cpu() &&
|
||||
index_weights_out.device().is_cpu(),
|
||||
"fused_indexer_q_rope_quant_cpu: all inputs must be CPU tensors");
|
||||
|
||||
const int64_t num_tokens = positions.size(0);
|
||||
const int64_t num_heads = index_q.size(1);
|
||||
const int64_t head_dim = index_q.size(2);
|
||||
const int64_t half_rot_dim = index_q_cos_sin_cache.size(-1) / 2;
|
||||
// Only ever called with head_dim=128, rot_dim=64 (DeepseekV4's sparse
|
||||
// indexer Q side); the AVX512 loops below process 16 elements/pairs per
|
||||
// iteration with no scalar remainder, so require even divisibility.
|
||||
TORCH_CHECK(head_dim % 16 == 0 && (head_dim - 2 * half_rot_dim) % 16 == 0 &&
|
||||
half_rot_dim % 16 == 0,
|
||||
"fused_indexer_q_rope_quant_cpu: head_dim, head_dim-rot_dim, and "
|
||||
"rot_dim/2 "
|
||||
"must be multiples of 16");
|
||||
|
||||
fused_indexer_q_rope_quant_impl(
|
||||
positions.data_ptr<int64_t>(), index_q.data_ptr<float>(),
|
||||
index_q.stride(0), index_q.stride(1),
|
||||
index_q_cos_sin_cache.data_ptr<float>(), index_q_cos_sin_cache.stride(0),
|
||||
half_rot_dim, index_q_fp8.data_ptr<at::Float8_e4m3fn>(),
|
||||
index_q_fp8.stride(0), index_q_fp8.stride(1), head_dim,
|
||||
index_weights.data_ptr<float>(), index_weights.stride(0),
|
||||
static_cast<float>(index_weights_softmax_scale),
|
||||
static_cast<float>(index_weights_head_scale),
|
||||
index_weights_out.data_ptr<float>(), index_weights_out.stride(0),
|
||||
num_tokens, num_heads);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,14 +1,14 @@
|
||||
// vLLM-native CPU cache-write op for MLA's single-latent-buffer KV cache.
|
||||
// `concat_and_cache_mla` (the generic op used by every GPU backend) is
|
||||
// CUDA-only; this is the CPU counterpart, adapted in spirit from SGLang's
|
||||
// `store_cache_cpu` (csrc/cpu/kvcache.cpp).
|
||||
//
|
||||
// `concat_and_cache_mla` (the generic MLA cache-write op used by every GPU
|
||||
// backend) is registered CUDA-only. This is the CPU counterpart, adapted in
|
||||
// spirit from SGLang's `store_cache_cpu` (csrc/cpu/kvcache.cpp) but
|
||||
// generalized to write two source tensors (`kv_c_normed`, `k_pe`) into two
|
||||
// different column-offset ranges of the SAME destination row -- SGLang's
|
||||
// version assumes k/v land in two independent, equal-row-width cache
|
||||
// tensors, which doesn't hold here since MLA's cache is one 576-wide buffer
|
||||
// and the two column ranges (512-wide, 64-wide) don't match the buffer's
|
||||
// true per-token stride, so the write can't reuse `store_cache_cpu` as-is.
|
||||
// Difference from upstream: writes two source tensors (`kv_c_normed`,
|
||||
// `k_pe`) into two column-offset ranges of the SAME destination row.
|
||||
// Upstream assumes k/v land in two independent, equal-row-width cache
|
||||
// tensors -- MLA's cache is one 576-wide buffer whose two column ranges
|
||||
// (512-wide, 64-wide) don't match that per-token stride, so
|
||||
// `store_cache_cpu` can't be reused as-is.
|
||||
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
|
||||
@@ -871,13 +871,13 @@ static inline void check_moe_scales(
|
||||
// topk_ids: [M, topk] (int32_t)
|
||||
//
|
||||
|
||||
at::Tensor fused_experts_cpu(
|
||||
void fused_experts_cpu(
|
||||
at::Tensor& out,
|
||||
at::Tensor& hidden_states,
|
||||
at::Tensor& w1,
|
||||
at::Tensor& w2,
|
||||
at::Tensor& topk_weights,
|
||||
at::Tensor& topk_ids,
|
||||
bool inplace,
|
||||
int64_t moe_comp_method,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
@@ -943,7 +943,7 @@ at::Tensor fused_experts_cpu(
|
||||
// check scales
|
||||
check_moe_scales(moe_comp_method, w1_scale, w2_scale, block_size);
|
||||
|
||||
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
|
||||
at::Tensor& out_hidden_states = out;
|
||||
|
||||
// NB: worst case is each expert holds a block with remainder of 1
|
||||
// 1. sorted_ids : [M * topk + E * (BLOCK_M - 1)]
|
||||
@@ -1214,7 +1214,6 @@ at::Tensor fused_experts_cpu(
|
||||
with_bias);
|
||||
}
|
||||
});
|
||||
return out_hidden_states;
|
||||
}
|
||||
|
||||
// shared expert kernel
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu:
|
||||
// `fp8_paged_mqa_logits_cpu`/`fp8_paged_mqa_logits_cpu_impl` (DECODE path).
|
||||
//
|
||||
// Reads the paged indexer K-cache directly via `page_table` -- no separate
|
||||
// eager gather step -- and processes the whole decode batch (one query token
|
||||
// per request; the CPU indexer decode path doesn't support native MTP/spec
|
||||
// decode yet, see `sparse_attn_indexer_cpu`) in a single `at::parallel_for`
|
||||
// flattening `(batch, token)`, exactly like upstream. AVX512 `dot_fp8_128`
|
||||
// dequant-dot-product and the split K-region-then-scale-region page byte
|
||||
// layout are verbatim (that part of the upstream contract already matches
|
||||
// vLLM's cache, written by `compress_norm_rope_store_indexer_cpu` in
|
||||
// compressor.cpp).
|
||||
//
|
||||
// Differences from upstream:
|
||||
// - `block_size` is a runtime argument instead of the hardcoded
|
||||
// `kBlockSize = 64`. vLLM's indexer K-cache shares the model's single
|
||||
// global `--block-size` KV-cache-group setting (confirmed empirically:
|
||||
// the DeepSeek-V4-Flash TP1/TP4 baseline configs run with
|
||||
// `block_size=256`, not 64 -- see `.agent-workspace/perf_baseline_tp*_
|
||||
// server.log`), so the upstream `kvcache_fp8.size(1) == 64` shape
|
||||
// contract does not hold in general.
|
||||
// - `q_fp8`/`weight` drop the redundant `S_q=1` dimension (`[batch, heads,
|
||||
// 128]`/`[batch, heads]` instead of `[batch, 1, heads, 128]`) since S_q is
|
||||
// always 1 here; `weight` is fp32-only (already folds q_scale/softmax_
|
||||
// scale/head_scale, matching this call site's actual dtype -- unlike
|
||||
// upstream's bf16/fp16/fp32 dispatch, which vLLM's CPU indexer never
|
||||
// produces).
|
||||
// - `index_head_dim` (128) stays a compile-time constant, matching
|
||||
// upstream: it's `config.index_head_dim`, a genuine DeepSeek-V4 model
|
||||
// invariant, not a vLLM KV-cache config knob like `block_size`.
|
||||
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
|
||||
#if !defined(CPU_CAPABILITY_AVX512)
|
||||
static_assert(false,
|
||||
"paged_mqa_logits.cpp is only ever compiled into the "
|
||||
"AVX512+AMX-flagged _C target (see cmake/cpu_extension.cmake's "
|
||||
"VLLM_EXT_SRC_SGL) and relies on that unconditionally -- "
|
||||
"CPU_CAPABILITY_AVX512 must be defined here.");
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int64_t kIndexHeadDim = 128;
|
||||
constexpr int64_t kIndexHeadDimWithScaleBytes = 132;
|
||||
|
||||
inline float dot_fp8_128(const uint8_t* k, const uint8_t* q) {
|
||||
__m512 acc = _mm512_setzero_ps();
|
||||
for (int64_t d = 0; d < kIndexHeadDim; d += 32) {
|
||||
const __m256i k8 =
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(k + d));
|
||||
const __m256i q8 =
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(q + d));
|
||||
acc = _mm512_dpbf16_ps(acc, CVT_FP8_TO_BF16(k8), CVT_FP8_TO_BF16(q8));
|
||||
}
|
||||
return _mm512_reduce_add_ps(acc);
|
||||
}
|
||||
|
||||
template <typename seq_t, typename page_t>
|
||||
void fp8_paged_mqa_logits_cpu_impl(const at::Tensor& q_fp8,
|
||||
const at::Tensor& kvcache_fp8,
|
||||
const at::Tensor& weight,
|
||||
const at::Tensor& seq_lens,
|
||||
const at::Tensor& page_table,
|
||||
at::Tensor& logits, int64_t block_size,
|
||||
int64_t max_seq_len) {
|
||||
const int64_t batch_size = q_fp8.size(0);
|
||||
const int64_t num_heads = q_fp8.size(1);
|
||||
const int64_t num_pages = kvcache_fp8.size(0);
|
||||
// Row stride is threaded through explicitly rather than assumed equal to
|
||||
// the page byte width: the K-cache may be a page of a larger pooled
|
||||
// allocation shared across layers (matches store_cache.cpp/flash_mla.cpp's
|
||||
// precedent).
|
||||
const int64_t page_stride = kvcache_fp8.stride(0);
|
||||
const int64_t pages_per_batch = page_table.size(1);
|
||||
const int64_t scale_offset_bytes = block_size * kIndexHeadDim;
|
||||
|
||||
const auto* q_ptr = reinterpret_cast<const uint8_t*>(q_fp8.const_data_ptr());
|
||||
const auto* cache_ptr =
|
||||
reinterpret_cast<const uint8_t*>(kvcache_fp8.const_data_ptr());
|
||||
const auto* weight_ptr = weight.const_data_ptr<float>();
|
||||
const auto* seq_ptr = seq_lens.const_data_ptr<seq_t>();
|
||||
const auto* page_ptr = page_table.const_data_ptr<page_t>();
|
||||
auto* out_ptr = logits.data_ptr<float>();
|
||||
|
||||
at::parallel_for(
|
||||
0, batch_size * max_seq_len, GRAIN_SIZE / kIndexHeadDim,
|
||||
[&](int64_t begin, int64_t end) {
|
||||
int64_t b{0}, token{0};
|
||||
data_index_init(begin, b, batch_size, token, max_seq_len);
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
const int64_t seq_len = static_cast<int64_t>(seq_ptr[b]);
|
||||
|
||||
if (token >= seq_len) {
|
||||
data_index_step(b, batch_size, token, max_seq_len);
|
||||
continue;
|
||||
}
|
||||
|
||||
const int64_t q_batch_offset = b * num_heads * kIndexHeadDim;
|
||||
const int64_t weight_batch_offset = b * num_heads;
|
||||
const int64_t page_batch_offset = b * pages_per_batch;
|
||||
float* out_row = out_ptr + b * max_seq_len;
|
||||
|
||||
const int64_t logical_page = token / block_size;
|
||||
const int64_t token_in_page = token % block_size;
|
||||
|
||||
const int64_t physical_page =
|
||||
static_cast<int64_t>(page_ptr[page_batch_offset + logical_page]);
|
||||
|
||||
const uint8_t* block = cache_ptr + physical_page * page_stride;
|
||||
const uint8_t* k_token = block + token_in_page * kIndexHeadDim;
|
||||
const float* scale_ptr =
|
||||
reinterpret_cast<const float*>(block + scale_offset_bytes);
|
||||
const float k_scale = scale_ptr[token_in_page];
|
||||
|
||||
float score_sum = 0.0f;
|
||||
for (int64_t h = 0; h < num_heads; ++h) {
|
||||
const uint8_t* q_head = q_ptr + q_batch_offset + h * kIndexHeadDim;
|
||||
float dot = dot_fp8_128(k_token, q_head);
|
||||
dot = std::max(dot, 0.0f);
|
||||
score_sum += dot * weight_ptr[weight_batch_offset + h];
|
||||
}
|
||||
|
||||
out_row[token] = score_sum * k_scale;
|
||||
data_index_step(b, batch_size, token, max_seq_len);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
template <typename seq_t>
|
||||
void dispatch_page_type(const at::Tensor& q_fp8, const at::Tensor& kvcache_fp8,
|
||||
const at::Tensor& weight, const at::Tensor& seq_lens,
|
||||
const at::Tensor& page_table, at::Tensor& logits,
|
||||
int64_t block_size, int64_t max_seq_len) {
|
||||
if (page_table.scalar_type() == at::kInt) {
|
||||
fp8_paged_mqa_logits_cpu_impl<seq_t, int32_t>(q_fp8, kvcache_fp8, weight,
|
||||
seq_lens, page_table, logits,
|
||||
block_size, max_seq_len);
|
||||
} else if (page_table.scalar_type() == at::kLong) {
|
||||
fp8_paged_mqa_logits_cpu_impl<seq_t, int64_t>(q_fp8, kvcache_fp8, weight,
|
||||
seq_lens, page_table, logits,
|
||||
block_size, max_seq_len);
|
||||
} else {
|
||||
TORCH_CHECK(false, "page_table must be int32 or int64");
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
at::Tensor fp8_paged_mqa_logits_cpu(at::Tensor& q_fp8, at::Tensor& kvcache_fp8,
|
||||
at::Tensor& weight, at::Tensor& seq_lens,
|
||||
at::Tensor& page_table, int64_t block_size,
|
||||
int64_t max_seq_len) {
|
||||
CHECK_INPUT(q_fp8);
|
||||
CHECK_CPU(kvcache_fp8);
|
||||
CHECK_INPUT(weight);
|
||||
CHECK_INPUT(seq_lens);
|
||||
CHECK_INPUT(page_table);
|
||||
TORCH_CHECK(
|
||||
q_fp8.dim() == 3 && q_fp8.scalar_type() == at::kFloat8_e4m3fn,
|
||||
"fp8_paged_mqa_logits_cpu: q_fp8 must be a 3D float8_e4m3fn tensor "
|
||||
"[batch, heads, 128]");
|
||||
TORCH_CHECK(q_fp8.size(2) == kIndexHeadDim,
|
||||
"fp8_paged_mqa_logits_cpu: q_fp8 head_dim must be 128");
|
||||
TORCH_CHECK(kvcache_fp8.dim() == 2 && kvcache_fp8.scalar_type() == at::kByte,
|
||||
"fp8_paged_mqa_logits_cpu: kvcache_fp8 must be a 2D uint8 tensor "
|
||||
"[num_pages, buf_numel_per_page]");
|
||||
TORCH_CHECK(kvcache_fp8.stride(1) == 1 &&
|
||||
kvcache_fp8.stride(0) >= kvcache_fp8.size(1),
|
||||
"fp8_paged_mqa_logits_cpu: kvcache_fp8 must be contiguous "
|
||||
"within a page (may be a page of a larger pooled allocation "
|
||||
"between pages)");
|
||||
TORCH_CHECK(block_size > 0,
|
||||
"fp8_paged_mqa_logits_cpu: block_size must be positive");
|
||||
TORCH_CHECK(
|
||||
kvcache_fp8.size(1) >= block_size * kIndexHeadDimWithScaleBytes,
|
||||
"fp8_paged_mqa_logits_cpu: kvcache_fp8 page byte width too small for "
|
||||
"block_size");
|
||||
|
||||
const int64_t batch_size = q_fp8.size(0);
|
||||
const int64_t num_heads = q_fp8.size(1);
|
||||
TORCH_CHECK(
|
||||
weight.dim() == 2 && weight.scalar_type() == at::kFloat &&
|
||||
weight.size(0) == batch_size && weight.size(1) == num_heads,
|
||||
"fp8_paged_mqa_logits_cpu: weight must be a 2D fp32 tensor [batch, "
|
||||
"heads]");
|
||||
TORCH_CHECK(seq_lens.dim() == 1 && seq_lens.size(0) == batch_size,
|
||||
"fp8_paged_mqa_logits_cpu: seq_lens must be a 1D tensor [batch]");
|
||||
TORCH_CHECK(page_table.dim() == 2 && page_table.size(0) == batch_size,
|
||||
"fp8_paged_mqa_logits_cpu: page_table must be a 2D tensor "
|
||||
"[batch, pages]");
|
||||
TORCH_CHECK(max_seq_len >= 0,
|
||||
"fp8_paged_mqa_logits_cpu: max_seq_len must be non-negative");
|
||||
|
||||
// Positions with token >= seq_len are never written by the kernel below,
|
||||
// and topk_transform_512_cpu (the only consumer) never reads them either
|
||||
// -- left uninitialized, matching upstream.
|
||||
auto logits =
|
||||
at::empty({batch_size, max_seq_len}, q_fp8.options().dtype(at::kFloat));
|
||||
if (batch_size == 0 || max_seq_len == 0) {
|
||||
return logits;
|
||||
}
|
||||
|
||||
if (seq_lens.scalar_type() == at::kInt) {
|
||||
dispatch_page_type<int32_t>(q_fp8, kvcache_fp8, weight, seq_lens,
|
||||
page_table, logits, block_size, max_seq_len);
|
||||
} else if (seq_lens.scalar_type() == at::kLong) {
|
||||
dispatch_page_type<int64_t>(q_fp8, kvcache_fp8, weight, seq_lens,
|
||||
page_table, logits, block_size, max_seq_len);
|
||||
} else {
|
||||
TORCH_CHECK(false, "seq_lens must be int32 or int64");
|
||||
}
|
||||
|
||||
return logits;
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/blob/main/python/sglang/kernels/aot/csrc/cpu/store_cache.cpp
|
||||
//
|
||||
// DeepSeek-V4 fused Q RMSNorm+RoPE / KV RoPE+UE8M0-quant+paged-cache-insert
|
||||
// kernel for the fp8_ds_mla cache layout. Not a port of upstream's
|
||||
// `set_k_and_s_cpu`/`quant_to_nope_fp8_rope_bf16_pack_cpu` -- their
|
||||
// pool_kv/pool_score signature doesn't apply to vLLM's layout. This is the
|
||||
// CPU counterpart to vLLM's own CUDA kernel
|
||||
// (`fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu`); see
|
||||
// `DeepseekV4Attention._fused_qnorm_rope_kv_insert_cpu` (attention.py) for
|
||||
// the eager reference this replaces.
|
||||
//
|
||||
// clang-format off
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
|
||||
#if !defined(CPU_CAPABILITY_AVX512)
|
||||
static_assert(false,
|
||||
"store_cache.cpp is only ever compiled into the AVX512+AMX-flagged "
|
||||
"_C target (see cmake/cpu_extension.cmake's VLLM_EXT_SRC_SGL) and "
|
||||
"relies on that unconditionally -- CPU_CAPABILITY_AVX512 must be "
|
||||
"defined here.");
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int64_t kRopeDim = 64;
|
||||
constexpr int64_t kQuantBlock = 64;
|
||||
constexpr float kFp8Max = 448.0f;
|
||||
constexpr float kFp8Min = -448.0f;
|
||||
constexpr float kQuantEps = 1e-4f;
|
||||
|
||||
inline void apply_gptj_rope_inplace(float* __restrict__ x, const float* __restrict__ cos, const float* __restrict__ sin, int64_t half_dim) {
|
||||
for (int64_t i = 0; i < half_dim; ++i) {
|
||||
const float e = x[2 * i];
|
||||
const float o = x[2 * i + 1];
|
||||
x[2 * i] = e * cos[i] - o * sin[i];
|
||||
x[2 * i + 1] = e * sin[i] + o * cos[i];
|
||||
}
|
||||
}
|
||||
|
||||
// AVX512 tile-quantization, adapted verbatim from SGLang's
|
||||
// `store_cache.cpp` (`quant_to_nope_fp8_rope_bf16_pack_cpu`'s
|
||||
// `cvt_fp32x16_to_fp8x16`/`quantize_tile_avx512`): per-64-element tile,
|
||||
// computes the UE8M0 power-of-two scale and quantizes to fp8_e4m3fn with
|
||||
// RNE rounding via bit manipulation instead of a per-element scalar
|
||||
// `at::Float8_e4m3fn` cast. Subnormal-magnitude fp8 lanes are flushed to
|
||||
// zero (matches SGLang's own convention, `cvt_fp32x16_to_fp8x16`) --
|
||||
// vLLM's own AVX512 dequant of this cache (`flash_mla.cpp`'s
|
||||
// `CVT_FP8_TO_BF16_EXT`) doesn't reconstruct exact subnormal magnitudes
|
||||
// either way, so there is nothing to preserve by rounding them exactly on
|
||||
// the write side.
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
|
||||
// Vectorized float32x16 -> fp8_e4m3fn x16 using AVX512 bit manipulation.
|
||||
// Implements RNE (round-to-nearest-even). Flushes subnormal fp8 to zero.
|
||||
// fp8_e4m3fn: sign(1), exp(4, bias=7), mant(3), no inf/nan special values.
|
||||
// Normal: exp_field in [1..15], value = 2^(exp-7) * (1 + mant/8)
|
||||
// Input assumed clamped to [-448, 448].
|
||||
inline __m128i cvt_fp32x16_to_fp8x16(__m512 input) {
|
||||
__m512i bits = _mm512_castps_si512(input);
|
||||
|
||||
// Extract sign -> bit 7 of fp8 byte
|
||||
__m512i signs = _mm512_srli_epi32(_mm512_and_si512(bits, _mm512_set1_epi32(0x80000000)), 24);
|
||||
|
||||
// Absolute value bits
|
||||
__m512i abs_bits = _mm512_and_si512(bits, _mm512_set1_epi32(0x7FFFFFFF));
|
||||
|
||||
// Float32 biased exponent
|
||||
__m512i f32_exp = _mm512_srli_epi32(abs_bits, 23);
|
||||
|
||||
// Float32 mantissa (23 bits)
|
||||
__m512i f32_mant = _mm512_and_si512(abs_bits, _mm512_set1_epi32(0x7FFFFF));
|
||||
|
||||
// fp8_exp = f32_exp - 120 (rebias from 127 to 7)
|
||||
__m512i fp8_exp = _mm512_sub_epi32(f32_exp, _mm512_set1_epi32(120));
|
||||
|
||||
// Top 3 mantissa bits
|
||||
__m512i mant3 = _mm512_srli_epi32(f32_mant, 20);
|
||||
|
||||
// RNE rounding: round_bit = bit 19, sticky = bits[18:0]
|
||||
__m512i round_bit = _mm512_and_si512(_mm512_srli_epi32(f32_mant, 19), _mm512_set1_epi32(1));
|
||||
__m512i sticky_bits = _mm512_and_si512(f32_mant, _mm512_set1_epi32(0x7FFFF));
|
||||
__mmask16 has_sticky = _mm512_cmpneq_epi32_mask(sticky_bits, _mm512_setzero_si512());
|
||||
|
||||
// Round up if round_bit AND (sticky OR lsb_of_mant3)
|
||||
__m512i lsb = _mm512_and_si512(mant3, _mm512_set1_epi32(1));
|
||||
__m512i sticky_or_lsb = _mm512_or_si512(_mm512_maskz_mov_epi32(has_sticky, _mm512_set1_epi32(1)), lsb);
|
||||
__m512i do_round = _mm512_and_si512(round_bit, sticky_or_lsb);
|
||||
mant3 = _mm512_add_epi32(mant3, do_round);
|
||||
|
||||
// Mantissa overflow (mant3 == 8) -> increment exponent, zero mantissa
|
||||
__mmask16 mant_ovf = _mm512_cmpeq_epi32_mask(mant3, _mm512_set1_epi32(8));
|
||||
mant3 = _mm512_mask_mov_epi32(mant3, mant_ovf, _mm512_setzero_si512());
|
||||
fp8_exp = _mm512_mask_add_epi32(fp8_exp, mant_ovf, fp8_exp, _mm512_set1_epi32(1));
|
||||
|
||||
// Clamp exponent: max 15
|
||||
fp8_exp = _mm512_min_epi32(fp8_exp, _mm512_set1_epi32(15));
|
||||
|
||||
// Result: (fp8_exp << 3) | mant3
|
||||
__m512i result = _mm512_or_si512(_mm512_slli_epi32(fp8_exp, 3), mant3);
|
||||
|
||||
// Flush to zero: if f32_exp < 121 (would be subnormal in fp8), output 0
|
||||
__mmask16 is_subnormal_or_zero = _mm512_cmplt_epi32_mask(f32_exp, _mm512_set1_epi32(121));
|
||||
result = _mm512_mask_mov_epi32(result, is_subnormal_or_zero, _mm512_setzero_si512());
|
||||
|
||||
// Add sign
|
||||
result = _mm512_or_si512(result, signs);
|
||||
|
||||
// Pack 16 x i32 -> 16 x i8
|
||||
return _mm512_cvtepi32_epi8(result);
|
||||
}
|
||||
|
||||
// Process one tile (64 bf16 values): find amax, compute scale, quantize to fp8.
|
||||
inline uint8_t quantize_tile_avx512(const at::BFloat16* __restrict__ src, uint8_t* __restrict__ dst) {
|
||||
// Load 64 bf16 -> 4 x 16 floats
|
||||
__m256i bf16_0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src));
|
||||
__m256i bf16_1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 16));
|
||||
__m256i bf16_2 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 32));
|
||||
__m256i bf16_3 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + 48));
|
||||
|
||||
__m512 f0 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_0), 16));
|
||||
__m512 f1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_1), 16));
|
||||
__m512 f2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_2), 16));
|
||||
__m512 f3 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(bf16_3), 16));
|
||||
|
||||
// Absolute values
|
||||
const __m512i abs_mask_i = _mm512_set1_epi32(0x7FFFFFFF);
|
||||
__m512 abs0 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f0), abs_mask_i));
|
||||
__m512 abs1 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f1), abs_mask_i));
|
||||
__m512 abs2 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f2), abs_mask_i));
|
||||
__m512 abs3 = _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(f3), abs_mask_i));
|
||||
|
||||
// Horizontal max
|
||||
__m512 max01 = _mm512_max_ps(abs0, abs1);
|
||||
__m512 max23 = _mm512_max_ps(abs2, abs3);
|
||||
__m512 max0123 = _mm512_max_ps(max01, max23);
|
||||
float amax = _mm512_reduce_max_ps(max0123);
|
||||
|
||||
// Scale computation
|
||||
float scale = std::max(amax / kFp8Max, kQuantEps);
|
||||
float ceil_log2 = std::ceil(std::log2(scale));
|
||||
float scale_pow2 = std::exp2(ceil_log2);
|
||||
float scale_inv = 1.0f / scale_pow2;
|
||||
|
||||
int exponent = static_cast<int>(ceil_log2);
|
||||
uint8_t scale_uint8 = static_cast<uint8_t>(exponent + 127);
|
||||
|
||||
// Scale all values
|
||||
__m512 vinv = _mm512_set1_ps(scale_inv);
|
||||
__m512 s0 = _mm512_mul_ps(f0, vinv);
|
||||
__m512 s1 = _mm512_mul_ps(f1, vinv);
|
||||
__m512 s2 = _mm512_mul_ps(f2, vinv);
|
||||
__m512 s3 = _mm512_mul_ps(f3, vinv);
|
||||
|
||||
// Clamp
|
||||
__m512 vmax = _mm512_set1_ps(kFp8Max);
|
||||
__m512 vmin = _mm512_set1_ps(kFp8Min);
|
||||
s0 = _mm512_max_ps(_mm512_min_ps(s0, vmax), vmin);
|
||||
s1 = _mm512_max_ps(_mm512_min_ps(s1, vmax), vmin);
|
||||
s2 = _mm512_max_ps(_mm512_min_ps(s2, vmax), vmin);
|
||||
s3 = _mm512_max_ps(_mm512_min_ps(s3, vmax), vmin);
|
||||
|
||||
// Vectorized float -> fp8 conversion (16 values at a time)
|
||||
__m128i fp8_0 = cvt_fp32x16_to_fp8x16(s0);
|
||||
__m128i fp8_1 = cvt_fp32x16_to_fp8x16(s1);
|
||||
__m128i fp8_2 = cvt_fp32x16_to_fp8x16(s2);
|
||||
__m128i fp8_3 = cvt_fp32x16_to_fp8x16(s3);
|
||||
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), fp8_0);
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), fp8_1);
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), fp8_2);
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), fp8_3);
|
||||
|
||||
return scale_uint8;
|
||||
}
|
||||
|
||||
#endif // CPU_CAPABILITY_AVX512
|
||||
|
||||
template <typename scalar_t>
|
||||
static void qnorm_rope_kv_insert_impl(
|
||||
scalar_t* __restrict__ q_out, // [num_tokens_full, q_head_padded, head_dim]
|
||||
uint8_t* __restrict__ cache, // [num_blocks, cache_row_stride] bytes
|
||||
const scalar_t* __restrict__ q, // [num_tokens_full, num_heads_q, head_dim]
|
||||
const scalar_t* __restrict__ kv, // [>=num_tokens_insert, head_dim]
|
||||
const int64_t* __restrict__ positions, // [num_tokens_full]
|
||||
const int64_t* __restrict__ slot_mapping, // [num_tokens_insert]
|
||||
const float* __restrict__ cos_sin_cache, // [max_pos, rope_dim] (first half cos, second half sin)
|
||||
int64_t num_tokens_full,
|
||||
int64_t num_heads_q,
|
||||
int64_t q_head_padded,
|
||||
int64_t head_dim,
|
||||
int64_t nope_dim,
|
||||
int64_t num_quant_blocks,
|
||||
int64_t token_data_bytes,
|
||||
int64_t scale_bytes_per_token,
|
||||
int64_t num_tokens_insert,
|
||||
int64_t cache_block_size,
|
||||
int64_t cache_row_stride,
|
||||
float eps) {
|
||||
const int64_t rope_half = kRopeDim / 2;
|
||||
|
||||
// Q: per-head weight-free RMSNorm + GPT-J RoPE on the last rope_dim dims.
|
||||
at::parallel_for(0, num_tokens_full * num_heads_q, 0, [&](int64_t begin, int64_t end) {
|
||||
std::vector<float> buf(head_dim);
|
||||
for (int64_t idx = begin; idx < end; ++idx) {
|
||||
const int64_t t = idx / num_heads_q;
|
||||
const int64_t h = idx % num_heads_q;
|
||||
const scalar_t* q_th = q + (t * num_heads_q + h) * head_dim;
|
||||
|
||||
float sq_sum = 0.f;
|
||||
for (int64_t i = 0; i < head_dim; ++i) {
|
||||
const float v = static_cast<float>(q_th[i]);
|
||||
buf[i] = v;
|
||||
sq_sum += v * v;
|
||||
}
|
||||
const float inv_rms = 1.0f / std::sqrt(sq_sum / static_cast<float>(head_dim) + eps);
|
||||
for (int64_t i = 0; i < head_dim; ++i)
|
||||
buf[i] *= inv_rms;
|
||||
|
||||
const int64_t pos = positions[t];
|
||||
const float* cos = cos_sin_cache + pos * kRopeDim;
|
||||
const float* sin = cos + rope_half;
|
||||
apply_gptj_rope_inplace(buf.data() + nope_dim, cos, sin, rope_half);
|
||||
|
||||
scalar_t* out_th = q_out + (t * q_head_padded + h) * head_dim;
|
||||
for (int64_t i = 0; i < head_dim; ++i)
|
||||
out_th[i] = static_cast<scalar_t>(buf[i]);
|
||||
}
|
||||
});
|
||||
|
||||
// KV: GPT-J RoPE (no RMSNorm) + bf16 round-trip + UE8M0 FP8 quant + paged insert.
|
||||
at::parallel_for(0, num_tokens_insert, 0, [&](int64_t begin, int64_t end) {
|
||||
std::vector<float> buf(head_dim);
|
||||
std::vector<at::BFloat16> nope_bf16(nope_dim);
|
||||
for (int64_t t = begin; t < end; ++t) {
|
||||
const int64_t slot_id = slot_mapping[t];
|
||||
if (slot_id < 0) continue;
|
||||
|
||||
const scalar_t* kv_t = kv + t * head_dim;
|
||||
for (int64_t i = 0; i < head_dim; ++i)
|
||||
buf[i] = static_cast<float>(kv_t[i]);
|
||||
|
||||
const int64_t pos = positions[t];
|
||||
const float* cos = cos_sin_cache + pos * kRopeDim;
|
||||
const float* sin = cos + rope_half;
|
||||
apply_gptj_rope_inplace(buf.data() + nope_dim, cos, sin, rope_half);
|
||||
|
||||
// bf16 round-trip, matching the CUDA kernel's double convert.
|
||||
for (int64_t i = 0; i < head_dim; ++i)
|
||||
buf[i] = static_cast<float>(at::BFloat16(buf[i]));
|
||||
for (int64_t i = 0; i < nope_dim; ++i)
|
||||
nope_bf16[i] = at::BFloat16(buf[i]);
|
||||
|
||||
const int64_t block_idx = slot_id / cache_block_size;
|
||||
const int64_t pos_in_block = slot_id % cache_block_size;
|
||||
uint8_t* const block_base = cache + block_idx * cache_row_stride;
|
||||
uint8_t* const token_fp8_ptr = block_base + pos_in_block * token_data_bytes;
|
||||
uint8_t* const token_bf16_ptr = token_fp8_ptr + nope_dim;
|
||||
uint8_t* const token_scale_ptr =
|
||||
block_base + cache_block_size * token_data_bytes + pos_in_block * scale_bytes_per_token;
|
||||
|
||||
// AVX512 tile-quantization (SGLang's `quantize_tile_avx512`), one
|
||||
// 64-wide tile per iteration.
|
||||
for (int64_t qb = 0; qb < num_quant_blocks; ++qb) {
|
||||
const at::BFloat16* tile_src = nope_bf16.data() + qb * kQuantBlock;
|
||||
uint8_t* tile_dst = token_fp8_ptr + qb * kQuantBlock;
|
||||
token_scale_ptr[qb] = quantize_tile_avx512(tile_src, tile_dst);
|
||||
}
|
||||
token_scale_ptr[num_quant_blocks] = 0;
|
||||
|
||||
auto* out_rope = reinterpret_cast<at::BFloat16*>(token_bf16_ptr);
|
||||
for (int64_t i = 0; i < kRopeDim; ++i)
|
||||
out_rope[i] = at::BFloat16(buf[nope_dim + i]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
at::Tensor fused_qnorm_rope_kv_insert_cpu(
|
||||
at::Tensor& q,
|
||||
at::Tensor& kv,
|
||||
at::Tensor& positions,
|
||||
at::Tensor& swa_kv_cache_2d,
|
||||
at::Tensor& slot_mapping,
|
||||
at::Tensor& cos_sin_cache,
|
||||
int64_t q_head_padded,
|
||||
double eps,
|
||||
int64_t cache_block_size) {
|
||||
TORCH_CHECK(
|
||||
q.dim() == 3 && (q.scalar_type() == at::kBFloat16 || q.scalar_type() == at::kHalf),
|
||||
"fused_qnorm_rope_kv_insert_cpu: q must be bf16/fp16 [num_tokens, num_heads_q, head_dim]");
|
||||
TORCH_CHECK(kv.dim() == 2 && kv.scalar_type() == q.scalar_type(), "fused_qnorm_rope_kv_insert_cpu: kv dtype must match q");
|
||||
TORCH_CHECK(positions.scalar_type() == at::kLong, "fused_qnorm_rope_kv_insert_cpu: positions must be int64");
|
||||
TORCH_CHECK(slot_mapping.scalar_type() == at::kLong, "fused_qnorm_rope_kv_insert_cpu: slot_mapping must be int64");
|
||||
TORCH_CHECK(
|
||||
swa_kv_cache_2d.dim() == 2 && swa_kv_cache_2d.scalar_type() == at::kByte,
|
||||
"fused_qnorm_rope_kv_insert_cpu: swa_kv_cache_2d must be a 2D uint8 tensor");
|
||||
TORCH_CHECK(
|
||||
q.device().is_cpu() && kv.device().is_cpu() && positions.device().is_cpu() && slot_mapping.device().is_cpu() &&
|
||||
swa_kv_cache_2d.device().is_cpu() && cos_sin_cache.device().is_cpu(),
|
||||
"fused_qnorm_rope_kv_insert_cpu: all inputs must be CPU tensors");
|
||||
|
||||
const int64_t num_tokens_full = q.size(0);
|
||||
const int64_t num_heads_q = q.size(1);
|
||||
const int64_t head_dim = q.size(2);
|
||||
const int64_t nope_dim = head_dim - kRopeDim;
|
||||
TORCH_CHECK(nope_dim > 0 && nope_dim % kQuantBlock == 0, "fused_qnorm_rope_kv_insert_cpu: unexpected head_dim");
|
||||
const int64_t num_quant_blocks = nope_dim / kQuantBlock;
|
||||
const int64_t scale_bytes_per_token = num_quant_blocks + 1;
|
||||
const int64_t token_data_bytes = nope_dim + kRopeDim * 2;
|
||||
|
||||
TORCH_CHECK(q_head_padded >= num_heads_q, "fused_qnorm_rope_kv_insert_cpu: q_head_padded must be >= num_heads_q");
|
||||
TORCH_CHECK(kv.size(1) == head_dim, "fused_qnorm_rope_kv_insert_cpu: kv head_dim must match q");
|
||||
TORCH_CHECK(positions.numel() == num_tokens_full, "fused_qnorm_rope_kv_insert_cpu: positions must have num_tokens_full entries");
|
||||
|
||||
const int64_t num_tokens_insert = slot_mapping.size(0);
|
||||
TORCH_CHECK(kv.size(0) >= num_tokens_insert, "fused_qnorm_rope_kv_insert_cpu: kv must have >= num_tokens_insert rows");
|
||||
TORCH_CHECK(cache_block_size > 0, "fused_qnorm_rope_kv_insert_cpu: cache_block_size must be positive");
|
||||
TORCH_CHECK(
|
||||
cos_sin_cache.scalar_type() == at::kFloat && cos_sin_cache.is_contiguous(),
|
||||
"fused_qnorm_rope_kv_insert_cpu: cos_sin_cache must be float32 and "
|
||||
"contiguous (convert once in Python)");
|
||||
TORCH_CHECK(
|
||||
q.is_contiguous() && kv.is_contiguous() && positions.is_contiguous() && slot_mapping.is_contiguous(),
|
||||
"fused_qnorm_rope_kv_insert_cpu: q/kv/positions/slot_mapping must be contiguous");
|
||||
|
||||
// swa_kv_cache_2d is mutated in place; index via its real row stride
|
||||
// rather than assuming it equals cache_block_size * (token_data_bytes +
|
||||
// scale_bytes_per_token) -- a paged/pooled buffer's row stride is not
|
||||
// guaranteed to match the logical per-block byte count.
|
||||
const int64_t cache_row_stride = swa_kv_cache_2d.stride(0);
|
||||
|
||||
auto q_out = at::zeros({num_tokens_full, q_head_padded, head_dim}, q.options());
|
||||
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(q.scalar_type(), "fused_qnorm_rope_kv_insert_cpu", [&] {
|
||||
qnorm_rope_kv_insert_impl<scalar_t>(
|
||||
q_out.data_ptr<scalar_t>(),
|
||||
swa_kv_cache_2d.data_ptr<uint8_t>(),
|
||||
q.data_ptr<scalar_t>(),
|
||||
kv.data_ptr<scalar_t>(),
|
||||
positions.data_ptr<int64_t>(),
|
||||
slot_mapping.data_ptr<int64_t>(),
|
||||
cos_sin_cache.data_ptr<float>(),
|
||||
num_tokens_full,
|
||||
num_heads_q,
|
||||
q_head_padded,
|
||||
head_dim,
|
||||
nope_dim,
|
||||
num_quant_blocks,
|
||||
token_data_bytes,
|
||||
scale_bytes_per_token,
|
||||
num_tokens_insert,
|
||||
cache_block_size,
|
||||
cache_row_stride,
|
||||
static_cast<float>(eps));
|
||||
});
|
||||
|
||||
return q_out;
|
||||
}
|
||||
|
||||
// vLLM-native: inverse GPT-J RoPE for `_o_proj`'s output de-rotation (undoes
|
||||
// the query's own rotation on the attention output before the low-rank
|
||||
// wo_a/wo_b projection -- see the math writeup in cpu_sparse.py's `_o_proj`).
|
||||
// Upstream needs this same step (`fused_rope_inplace(..., inverse=True)` in
|
||||
// `deepseek_v4.py`) but only has a CUDA-JIT implementation, no CPU kernel --
|
||||
// this fills that gap rather than porting one. Replaces the eager reference
|
||||
// in `DeepseekV4CPUAttention._o_proj` (cpu_sparse.py). Uses the same
|
||||
// half-half `[cos...|sin...]` cache layout as `apply_gptj_rope_inplace`
|
||||
// above (not upstream's interleaved `[c0,s0,c1,s1,...]` layout).
|
||||
//
|
||||
// AVX512 path: NoPE dtype-convert copy uses `vec.h`'s `load_float_vec`
|
||||
// (16-wide bf16/fp16 -> fp32). The interleaved (even,odd) rotation widens
|
||||
// 32 raw elements to fp32 via `load_float_vec2`, round-trips them through
|
||||
// an aligned scratch buffer, then gathers/scatters the even/odd lanes --
|
||||
// same technique as `indexer.cpp`'s Q-side RoPE, since `cos`/`sin` here
|
||||
// are two contiguous halves rather than per-pair-interleaved.
|
||||
at::Tensor inverse_gptj_rope_o_proj_cpu(
|
||||
at::Tensor& o, // [num_tokens, num_heads, head_dim]
|
||||
at::Tensor& positions, // [num_tokens]
|
||||
at::Tensor& cos_sin_cache, // [max_pos, rope_dim] (first half cos, second half sin)
|
||||
int64_t rope_dim) {
|
||||
TORCH_CHECK(o.dim() == 3, "inverse_gptj_rope_o_proj_cpu: o must be 3D [num_tokens, num_heads, head_dim]");
|
||||
TORCH_CHECK(positions.scalar_type() == at::kLong, "inverse_gptj_rope_o_proj_cpu: positions must be int64");
|
||||
TORCH_CHECK(
|
||||
o.device().is_cpu() && positions.device().is_cpu() && cos_sin_cache.device().is_cpu(),
|
||||
"inverse_gptj_rope_o_proj_cpu: all inputs must be CPU tensors");
|
||||
|
||||
const int64_t num_tokens = o.size(0);
|
||||
const int64_t num_heads = o.size(1);
|
||||
const int64_t head_dim = o.size(2);
|
||||
const int64_t nope_dim = head_dim - rope_dim;
|
||||
const int64_t half_dim = rope_dim / 2;
|
||||
TORCH_CHECK(nope_dim >= 0, "inverse_gptj_rope_o_proj_cpu: rope_dim must not exceed head_dim");
|
||||
// Only ever called with head_dim=512, rope_dim=64 (DeepseekV4CPUAttention's
|
||||
// _o_proj); the AVX512 loops below process 16 elements/pairs per iteration
|
||||
// with no scalar remainder, so require both to divide evenly.
|
||||
TORCH_CHECK(nope_dim % 16 == 0 && half_dim % 16 == 0,
|
||||
"inverse_gptj_rope_o_proj_cpu: head_dim-rope_dim and rope_dim/2 must be multiples of 16");
|
||||
TORCH_CHECK(
|
||||
cos_sin_cache.scalar_type() == at::kFloat && cos_sin_cache.is_contiguous(),
|
||||
"inverse_gptj_rope_o_proj_cpu: cos_sin_cache must be float32 and "
|
||||
"contiguous (convert once in Python)");
|
||||
TORCH_CHECK(o.is_contiguous() && positions.is_contiguous(),
|
||||
"inverse_gptj_rope_o_proj_cpu: o/positions must be contiguous");
|
||||
|
||||
auto out = at::empty({num_tokens, num_heads, head_dim}, o.options().dtype(at::kFloat));
|
||||
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(o.scalar_type(), "inverse_gptj_rope_o_proj_cpu", [&] {
|
||||
const scalar_t* o_ptr = o.data_ptr<scalar_t>();
|
||||
float* out_ptr = out.data_ptr<float>();
|
||||
const int64_t* pos_ptr = positions.data_ptr<int64_t>();
|
||||
const float* cos_sin_ptr = cos_sin_cache.data_ptr<float>();
|
||||
|
||||
at::parallel_for(0, num_tokens * num_heads, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t idx = begin; idx < end; ++idx) {
|
||||
const int64_t t = idx / num_heads;
|
||||
const int64_t h = idx % num_heads;
|
||||
const scalar_t* src = o_ptr + (t * num_heads + h) * head_dim;
|
||||
float* dst = out_ptr + (t * num_heads + h) * head_dim;
|
||||
|
||||
// head_dim=512, rope_dim=64 in practice (checked above), so
|
||||
// nope_dim/half_dim are always multiples of 16 -- no scalar
|
||||
// remainder needed.
|
||||
for (int64_t i = 0; i < nope_dim; i += 16) {
|
||||
load_float_vec<scalar_t>(src + i).store(dst + i);
|
||||
}
|
||||
|
||||
const int64_t pos = pos_ptr[t];
|
||||
const float* cos = cos_sin_ptr + pos * rope_dim;
|
||||
const float* sin = cos + half_dim;
|
||||
|
||||
{
|
||||
// Permute-based (no gather/scatter), adapted from SGLang's
|
||||
// `rope.cpp` (`apply_rotary_emb_row`) the same way as
|
||||
// compressor.cpp's `apply_gptj_rope_inplace`: `_mm512_permute_ps`
|
||||
// swaps each pair's (even, odd) lanes in one shuffle, and cos/sin
|
||||
// -- loaded as 8 contiguous values -- are duplicated pairwise via
|
||||
// `_mm512_permutexvar_ps` to align with that layout, since our
|
||||
// cos/sin are two separate flat arrays rather than SGLang's
|
||||
// pre-interleaved freqs. Two batches of 8 pairs per iteration
|
||||
// (matching `load_float_vec2`'s 16+16-lane split).
|
||||
const __m512i dup_idx = _mm512_set_epi32(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0);
|
||||
// Inverse GPT-J rotation negates the sin broadcast on the odd
|
||||
// (imag) lane of each pair; the even (real) lane keeps sin's sign.
|
||||
const __m512 sign_mask = _mm512_castsi512_ps(_mm512_set_epi32(
|
||||
(int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0,
|
||||
(int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0, (int)0x80000000, 0));
|
||||
for (int64_t j = 0; j < half_dim; j += 16) {
|
||||
const scalar_t* rsrc = src + nope_dim + 2 * j;
|
||||
at::vec::Vectorized<float> x0, x1;
|
||||
std::tie(x0, x1) = load_float_vec2(rsrc);
|
||||
float* rdst = dst + nope_dim + 2 * j;
|
||||
|
||||
__m512 xv0 = x0;
|
||||
__m512 cos_b0 = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(cos + j)));
|
||||
__m512 sin_b0 = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(sin + j)));
|
||||
__m512 out0 = _mm512_fmadd_ps(
|
||||
xv0, cos_b0, _mm512_mul_ps(_mm512_permute_ps(xv0, 0xB1), _mm512_xor_ps(sin_b0, sign_mask)));
|
||||
_mm512_storeu_ps(rdst, out0);
|
||||
|
||||
__m512 xv1 = x1;
|
||||
__m512 cos_b1 = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(cos + j + 8)));
|
||||
__m512 sin_b1 = _mm512_permutexvar_ps(dup_idx, _mm512_castps256_ps512(_mm256_loadu_ps(sin + j + 8)));
|
||||
__m512 out1 = _mm512_fmadd_ps(
|
||||
xv1, cos_b1, _mm512_mul_ps(_mm512_permute_ps(xv1, 0xB1), _mm512_xor_ps(sin_b1, sign_mask)));
|
||||
_mm512_storeu_ps(rdst + 16, out1);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,834 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu:
|
||||
// `biased_topk_kernel_impl`/`biased_topk_cpu` (flat, non-grouped DeepSeek-V4
|
||||
// biased top-k) and `hash_topk_kernel_impl`/`hash_topk_cpu` (hash-routed
|
||||
// layers, expert IDs from a precomputed `tid2eid` lookup table), plus their
|
||||
// `sigmoid`/`softmax`/`apply_bias`/`sqrtsoftplus` scoring helpers. See
|
||||
// `biased_topk_cpu`/`hash_topk_cpu` below for the one deliberate deviation
|
||||
// from the upstream kernel (routed_scaling_factor handling when no shared
|
||||
// expert is fused into routing, vLLM's CPU DeepSeek-V4 calling convention).
|
||||
//
|
||||
// clang-format off
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
|
||||
namespace {
|
||||
|
||||
template <typename scalar_t, int SIZE, std::enable_if_t<!std::is_same_v<scalar_t, float>, int> = 0>
|
||||
inline void sigmoid(float* __restrict__ out, const scalar_t* __restrict__ input) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
|
||||
const fVec one = fVec(1.f);
|
||||
|
||||
constexpr int kVecSize = bVec::size();
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
bVec x_bvec = bVec::loadu(input + d);
|
||||
fVec x_fvec0, x_fvec1;
|
||||
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
|
||||
|
||||
x_fvec0 = one / (one + x_fvec0.neg().exp_u20());
|
||||
x_fvec1 = one / (one + x_fvec1.neg().exp_u20());
|
||||
|
||||
x_fvec0.store(out + d);
|
||||
x_fvec1.store(out + d + fVec::size());
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int SIZE, std::enable_if_t<std::is_same_v<scalar_t, float>, int> = 0>
|
||||
inline void sigmoid(float* __restrict__ out, const float* __restrict__ input) {
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
const fVec one = fVec(1.f);
|
||||
constexpr int kVecSize = fVec::size();
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
fVec in_fvec = fVec::loadu(input + d);
|
||||
in_fvec = one / (one + in_fvec.neg().exp_u20());
|
||||
in_fvec.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int SIZE, std::enable_if_t<!std::is_same_v<scalar_t, float>, int> = 0>
|
||||
inline void softmax(float* __restrict__ out, const scalar_t* __restrict__ input) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
|
||||
constexpr int kVecSize = bVec::size();
|
||||
|
||||
// step 1: get max
|
||||
fVec max_fvec = fVec(-std::numeric_limits<float>::infinity());
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
bVec x_bvec = bVec::loadu(input + d);
|
||||
fVec x_fvec0, x_fvec1;
|
||||
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
|
||||
|
||||
max_fvec = at::vec::maximum(max_fvec, x_fvec0);
|
||||
max_fvec = at::vec::maximum(max_fvec, x_fvec1);
|
||||
x_fvec0.store(out + d);
|
||||
x_fvec1.store(out + d + fVec::size());
|
||||
}
|
||||
float max_val = vec_reduce_max(max_fvec);
|
||||
max_fvec = fVec(max_val);
|
||||
|
||||
// step 2: sum of (x - max).exp()
|
||||
fVec sum_fvec = fVec(float(0));
|
||||
for (int d = 0; d < SIZE; d += fVec::size()) {
|
||||
fVec x_fvec = (fVec::loadu(out + d) - max_fvec).exp_u20();
|
||||
sum_fvec += x_fvec;
|
||||
x_fvec.store(out + d);
|
||||
}
|
||||
float sum_val = vec_reduce_sum(sum_fvec);
|
||||
|
||||
// step 3: x * (1 / sum)
|
||||
sum_fvec = fVec(1.f / sum_val);
|
||||
for (int d = 0; d < SIZE; d += fVec::size()) {
|
||||
fVec out_fvec = fVec::loadu(out + d) * sum_fvec;
|
||||
out_fvec.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int SIZE, std::enable_if_t<std::is_same_v<scalar_t, float>, int> = 0>
|
||||
inline void softmax(float* __restrict__ out, const float* __restrict__ input) {
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
|
||||
constexpr int kVecSize = fVec::size();
|
||||
|
||||
// step 1: get max
|
||||
fVec max_fvec = fVec(-std::numeric_limits<float>::infinity());
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
fVec x_fvec = fVec::loadu(input + d);
|
||||
max_fvec = at::vec::maximum(max_fvec, x_fvec);
|
||||
x_fvec.store(out + d);
|
||||
}
|
||||
float max_val = vec_reduce_max(max_fvec);
|
||||
max_fvec = fVec(max_val);
|
||||
|
||||
// step 2: sum of (x - max).exp()
|
||||
fVec sum_fvec = fVec(float(0));
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
fVec x_fvec = (fVec::loadu(out + d) - max_fvec).exp_u20();
|
||||
sum_fvec += x_fvec;
|
||||
x_fvec.store(out + d);
|
||||
}
|
||||
float sum_val = vec_reduce_sum(sum_fvec);
|
||||
|
||||
// step 3: x * (1 / sum)
|
||||
sum_fvec = fVec(1.f / sum_val);
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
fVec out_fvec = fVec::loadu(out + d) * sum_fvec;
|
||||
out_fvec.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename param_t, int SIZE>
|
||||
inline void
|
||||
apply_bias(float* __restrict__ scores2, const float* __restrict__ scores, const param_t* __restrict__ bias) {
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
auto vec_size = fVec::size() * 2;
|
||||
int d = 0;
|
||||
for (; d <= SIZE - vec_size; d += vec_size) {
|
||||
fVec bias0, bias1, x0, x1;
|
||||
std::tie(bias0, bias1) = load_float_vec2(bias + d);
|
||||
std::tie(x0, x1) = load_float_vec2(scores + d);
|
||||
x0 = x0 + bias0;
|
||||
x1 = x1 + bias1;
|
||||
x0.store(scores2 + d);
|
||||
x1.store(scores2 + d + fVec::size());
|
||||
}
|
||||
for (; d < SIZE; d++) {
|
||||
scores2[d] = scores[d] + (float)bias[d];
|
||||
}
|
||||
}
|
||||
|
||||
// sqrtsoftplus: sqrt(softplus(x)) = sqrt(log(1 + exp(x)))
|
||||
// For numerical stability: when x > threshold, softplus(x) ≈ x
|
||||
// When x < -threshold, softplus(x) ≈ exp(x), so sqrt(softplus(x)) ≈ sqrt(exp(x)) = exp(x/2)
|
||||
template <typename scalar_t, int SIZE, std::enable_if_t<!std::is_same_v<scalar_t, float>, int> = 0>
|
||||
inline void sqrtsoftplus(float* __restrict__ out, const scalar_t* __restrict__ input) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
|
||||
const fVec one = fVec(1.f);
|
||||
const fVec half = fVec(0.5f);
|
||||
const fVec threshold = fVec(20.f);
|
||||
const fVec neg_threshold = fVec(-15.f); // below this, 1+exp(x) loses precision in float32
|
||||
|
||||
constexpr int kVecSize = bVec::size();
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
bVec x_bvec = bVec::loadu(input + d);
|
||||
fVec x_fvec0, x_fvec1;
|
||||
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
|
||||
|
||||
// default: softplus(x) = log(1 + exp(x))
|
||||
fVec sp0 = (one + x_fvec0.exp_u20()).log();
|
||||
fVec sp1 = (one + x_fvec1.exp_u20()).log();
|
||||
// x > 20: softplus(x) ≈ x
|
||||
sp0 = fVec::blendv(sp0, x_fvec0, x_fvec0 > threshold);
|
||||
sp1 = fVec::blendv(sp1, x_fvec1, x_fvec1 > threshold);
|
||||
// x < -15: softplus(x) ≈ exp(x), sqrt(exp(x)) = exp(x/2)
|
||||
fVec exp_half0 = (x_fvec0 * half).exp_u20();
|
||||
fVec exp_half1 = (x_fvec1 * half).exp_u20();
|
||||
sp0 = fVec::blendv(sp0.sqrt(), exp_half0, x_fvec0 < neg_threshold);
|
||||
sp1 = fVec::blendv(sp1.sqrt(), exp_half1, x_fvec1 < neg_threshold);
|
||||
|
||||
sp0.store(out + d);
|
||||
sp1.store(out + d + fVec::size());
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int SIZE, std::enable_if_t<std::is_same_v<scalar_t, float>, int> = 0>
|
||||
inline void sqrtsoftplus(float* __restrict__ out, const float* __restrict__ input) {
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
const fVec one = fVec(1.f);
|
||||
const fVec half = fVec(0.5f);
|
||||
const fVec threshold = fVec(20.f);
|
||||
const fVec neg_threshold = fVec(-15.f);
|
||||
constexpr int kVecSize = fVec::size();
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
fVec x = fVec::loadu(input + d);
|
||||
fVec sp = (one + x.exp_u20()).log();
|
||||
sp = fVec::blendv(sp, x, x > threshold);
|
||||
fVec exp_half = (x * half).exp_u20();
|
||||
sp = fVec::blendv(sp.sqrt(), exp_half, x < neg_threshold);
|
||||
sp.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
// biased_topk: flat (non-grouped) biased topk for DeepSeek V4
|
||||
// scoring_func: 0 = sigmoid, 1 = sqrtsoftplus
|
||||
template <typename scalar_t, typename param_t, int NUM_EXPERTS, int TOPK>
|
||||
void biased_topk_kernel_impl(
|
||||
float* __restrict__ topk_weights,
|
||||
int32_t* __restrict__ topk_ids,
|
||||
const scalar_t* __restrict__ gating_output,
|
||||
const param_t* __restrict__ bias,
|
||||
int64_t num_tokens,
|
||||
int64_t topk,
|
||||
bool renormalize,
|
||||
int scoring_func,
|
||||
int64_t num_fused_shared_experts,
|
||||
float routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output) {
|
||||
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
|
||||
alignas(64) float scores[NUM_EXPERTS];
|
||||
alignas(64) float scores_biased[NUM_EXPERTS];
|
||||
|
||||
using elem_t = std::pair<float, int32_t>;
|
||||
std::array<elem_t, NUM_EXPERTS> queue;
|
||||
|
||||
// simple RNG for fused shared expert random ID
|
||||
uint64_t rng_state = begin * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
// compute scores
|
||||
if (scoring_func == 0) {
|
||||
sigmoid<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
|
||||
} else {
|
||||
sqrtsoftplus<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
|
||||
}
|
||||
|
||||
// add bias for selection
|
||||
apply_bias<param_t, NUM_EXPERTS>(scores_biased, scores, bias);
|
||||
|
||||
// build queue and partial sort to find top-k
|
||||
for (int64_t e = 0; e < NUM_EXPERTS; ++e) {
|
||||
queue[e] = {scores_biased[e], static_cast<int32_t>(e)};
|
||||
}
|
||||
|
||||
std::partial_sort(
|
||||
queue.begin(), queue.begin() + topk, queue.end(), [](const elem_t& x, const elem_t& y) -> bool {
|
||||
return x.first > y.first;
|
||||
});
|
||||
|
||||
// gather original scores (without bias) as weights
|
||||
for (int64_t j = 0; j < topk; ++j) {
|
||||
int32_t idx = queue[j].second;
|
||||
topk_ids[i * topk + j] = idx;
|
||||
topk_weights[i * topk + j] = scores[idx];
|
||||
}
|
||||
|
||||
// handle fused shared experts
|
||||
if (num_fused_shared_experts > 0) {
|
||||
// replace last slot with random shared expert ID
|
||||
rng_state = rng_state * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||
int32_t shared_id =
|
||||
NUM_EXPERTS + static_cast<int32_t>((rng_state >> 33) % static_cast<uint64_t>(num_fused_shared_experts));
|
||||
topk_ids[i * topk + topk - 1] = shared_id;
|
||||
|
||||
// shared expert weight = sum of routed weights / scaling_factor
|
||||
if (routed_scaling_factor != 0.0f) {
|
||||
float routed_sum = 0.f;
|
||||
for (int64_t j = 0; j < topk - 1; ++j) {
|
||||
routed_sum += topk_weights[i * topk + j];
|
||||
}
|
||||
topk_weights[i * topk + topk - 1] = routed_sum / routed_scaling_factor;
|
||||
}
|
||||
}
|
||||
|
||||
// renormalize
|
||||
if (renormalize) {
|
||||
float sum = 0.f;
|
||||
int64_t norm_end = (num_fused_shared_experts == 0) ? topk : topk - 1;
|
||||
for (int64_t j = 0; j < norm_end; ++j) {
|
||||
sum += topk_weights[i * topk + j];
|
||||
}
|
||||
float scale = 1.f / sum;
|
||||
if (apply_routed_scaling_factor_on_output) {
|
||||
scale *= routed_scaling_factor;
|
||||
}
|
||||
for (int64_t j = 0; j < norm_end; ++j) {
|
||||
topk_weights[i * topk + j] *= scale;
|
||||
}
|
||||
// also scale the shared expert weight if present
|
||||
if (num_fused_shared_experts > 0) {
|
||||
topk_weights[i * topk + topk - 1] *= scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// hash_topk: expert IDs come from a precomputed lookup table tid2eid[input_ids]
|
||||
// scoring_func: 0 = softmax, 1 = sigmoid, 2 = sqrtsoftplus
|
||||
template <typename scalar_t, int NUM_EXPERTS, int TOPK>
|
||||
void hash_topk_kernel_impl(
|
||||
float* __restrict__ topk_weights,
|
||||
int32_t* __restrict__ topk_ids,
|
||||
const scalar_t* __restrict__ gating_output,
|
||||
const int32_t* __restrict__ tid2eid, // [num_tokens, routed_topk]
|
||||
int64_t num_tokens,
|
||||
int scoring_func,
|
||||
int64_t num_fused_shared_experts,
|
||||
int64_t num_experts,
|
||||
float routed_scaling_factor,
|
||||
int64_t topk) {
|
||||
const int64_t routed_topk = topk - num_fused_shared_experts;
|
||||
const bool need_renormalize = (scoring_func != 0); // renormalize for non-softmax
|
||||
|
||||
at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) {
|
||||
alignas(64) float scores[NUM_EXPERTS];
|
||||
|
||||
// simple RNG for fused shared expert random ID
|
||||
uint64_t rng_state = begin * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
// compute scores over all experts
|
||||
if (scoring_func == 0) {
|
||||
softmax<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
|
||||
} else if (scoring_func == 1) {
|
||||
sigmoid<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
|
||||
} else {
|
||||
sqrtsoftplus<scalar_t, NUM_EXPERTS>(scores, gating_output + i * NUM_EXPERTS);
|
||||
}
|
||||
|
||||
// gather expert IDs from lookup table
|
||||
const int32_t* eid_row = tid2eid + i * routed_topk;
|
||||
for (int64_t j = 0; j < routed_topk; ++j) {
|
||||
int32_t eid = eid_row[j];
|
||||
topk_ids[i * topk + j] = eid;
|
||||
topk_weights[i * topk + j] = scores[eid];
|
||||
}
|
||||
|
||||
// renormalize routed weights (for non-softmax scoring). Unlike the
|
||||
// upstream kernel, always fold routed_scaling_factor in here rather
|
||||
// than only applying it to the fused-shared-expert slot below: vLLM's
|
||||
// DeepSeek-V4 CPU hash-routed layers never fuse a shared expert into
|
||||
// this routing table (num_fused_shared_experts is always 0 on this
|
||||
// call path), so this is the only place the scaling factor can be
|
||||
// applied. Matches the eager reference (`sqrtsoftplus_bias_topk` in
|
||||
// cpu_moe.py) and the CUDA kernel (`dsv4HashTopkSoftplusSqrt`).
|
||||
if (need_renormalize) {
|
||||
float sum = 0.f;
|
||||
for (int64_t j = 0; j < routed_topk; ++j) {
|
||||
sum += topk_weights[i * topk + j];
|
||||
}
|
||||
if (sum > 0.f) {
|
||||
float scale = 1.f / sum;
|
||||
if (num_fused_shared_experts == 0) {
|
||||
scale *= routed_scaling_factor;
|
||||
}
|
||||
for (int64_t j = 0; j < routed_topk; ++j) {
|
||||
topk_weights[i * topk + j] *= scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handle fused shared expert
|
||||
if (num_fused_shared_experts > 0) {
|
||||
// random shared expert ID in [num_experts, num_experts + num_fused_shared_experts)
|
||||
rng_state = rng_state * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||
int32_t shared_id =
|
||||
num_experts + static_cast<int32_t>((rng_state >> 33) % static_cast<uint64_t>(num_fused_shared_experts));
|
||||
topk_ids[i * topk + topk - 1] = shared_id;
|
||||
|
||||
// shared expert weight = sum of routed weights / scaling_factor
|
||||
float routed_sum = 0.f;
|
||||
for (int64_t j = 0; j < routed_topk; ++j) {
|
||||
routed_sum += topk_weights[i * topk + j];
|
||||
}
|
||||
topk_weights[i * topk + topk - 1] = routed_sum / routed_scaling_factor;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#define LAUNCH_HASH_TOPK_KERNEL(NE, NTOPK) \
|
||||
hash_topk_kernel_impl<scalar_t, NE, NTOPK>( \
|
||||
topk_weights.data_ptr<float>(), \
|
||||
topk_ids.data_ptr<int32_t>(), \
|
||||
gating_output.data_ptr<scalar_t>(), \
|
||||
tid2eid_flat.data_ptr<int32_t>(), \
|
||||
num_tokens, \
|
||||
scoring_func_id, \
|
||||
num_fused_shared_experts, \
|
||||
num_experts, \
|
||||
routed_scaling_factor_value, \
|
||||
topk);
|
||||
|
||||
#define LAUNCH_BIASED_TOPK_KERNEL(NE, NTOPK) \
|
||||
biased_topk_kernel_impl<scalar_t, param_t, NE, NTOPK>( \
|
||||
topk_weights.data_ptr<float>(), \
|
||||
topk_ids.data_ptr<int32_t>(), \
|
||||
gating_output.data_ptr<scalar_t>(), \
|
||||
correction_bias.data_ptr<param_t>(), \
|
||||
num_tokens, \
|
||||
topk, \
|
||||
renormalize, \
|
||||
scoring_func_id, \
|
||||
num_fused_shared_experts, \
|
||||
routed_scaling_factor_value, \
|
||||
apply_routed_scaling_factor_on_output);
|
||||
|
||||
} // namespace
|
||||
|
||||
// biased topk for DeepSeek V4 (flat, non-grouped)
|
||||
std::tuple<at::Tensor, at::Tensor> biased_topk_cpu(
|
||||
at::Tensor& hidden_states,
|
||||
at::Tensor& gating_output,
|
||||
at::Tensor& correction_bias,
|
||||
int64_t topk,
|
||||
bool renormalize,
|
||||
std::string scoring_func,
|
||||
int64_t num_fused_shared_experts,
|
||||
std::optional<double> routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output) {
|
||||
CHECK_INPUT(gating_output);
|
||||
CHECK_INPUT(correction_bias);
|
||||
|
||||
const auto st = gating_output.scalar_type();
|
||||
int64_t num_tokens = hidden_states.size(0);
|
||||
int64_t num_experts = gating_output.size(1);
|
||||
TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch");
|
||||
TORCH_CHECK(correction_bias.numel() == num_experts, "Bias shape mismatch");
|
||||
|
||||
int scoring_func_id = 0;
|
||||
if (scoring_func == "sigmoid") {
|
||||
scoring_func_id = 0;
|
||||
} else if (scoring_func == "sqrtsoftplus") {
|
||||
scoring_func_id = 1;
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported scoring_func: ", scoring_func);
|
||||
}
|
||||
|
||||
float routed_scaling_factor_value = routed_scaling_factor.has_value() ? routed_scaling_factor.value() : 0.0f;
|
||||
|
||||
at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat));
|
||||
at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt));
|
||||
|
||||
// The actual routed topk (excluding fused shared experts)
|
||||
int64_t routed_topk = topk - num_fused_shared_experts;
|
||||
|
||||
CPU_DISPATCH_FLOATING_TYPES_EXT(st, correction_bias.scalar_type(), "biased_topk_kernel", [&] {
|
||||
// dispatch on num_experts and routed_topk. For DeepSeek V4: 256 experts,
|
||||
// routed_topk=6 (num_fused_shared_experts is always 0 on this CPU call
|
||||
// path -- see cpu_moe.py). Anything outside this table isn't a real
|
||||
// model shape, so fail loudly instead of silently degrading to a slow
|
||||
// generic path.
|
||||
switch (num_experts) {
|
||||
case 64:
|
||||
switch (routed_topk) {
|
||||
case 6:
|
||||
LAUNCH_BIASED_TOPK_KERNEL(64, 6);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_BIASED_TOPK_KERNEL(64, 8);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "biased_topk_cpu: unsupported routed_topk ", routed_topk,
|
||||
" for num_experts=64");
|
||||
}
|
||||
break;
|
||||
case 128:
|
||||
switch (routed_topk) {
|
||||
case 6:
|
||||
LAUNCH_BIASED_TOPK_KERNEL(128, 6);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_BIASED_TOPK_KERNEL(128, 8);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "biased_topk_cpu: unsupported routed_topk ", routed_topk,
|
||||
" for num_experts=128");
|
||||
}
|
||||
break;
|
||||
case 256:
|
||||
switch (routed_topk) {
|
||||
case 6:
|
||||
LAUNCH_BIASED_TOPK_KERNEL(256, 6);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_BIASED_TOPK_KERNEL(256, 8);
|
||||
break;
|
||||
case 9:
|
||||
LAUNCH_BIASED_TOPK_KERNEL(256, 9);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "biased_topk_cpu: unsupported routed_topk ", routed_topk,
|
||||
" for num_experts=256");
|
||||
}
|
||||
break;
|
||||
case 384:
|
||||
switch (routed_topk) {
|
||||
case 6:
|
||||
LAUNCH_BIASED_TOPK_KERNEL(384, 6);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_BIASED_TOPK_KERNEL(384, 8);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "biased_topk_cpu: unsupported routed_topk ", routed_topk,
|
||||
" for num_experts=384");
|
||||
}
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "biased_topk_cpu: unsupported num_experts ", num_experts);
|
||||
}
|
||||
});
|
||||
return std::make_tuple(topk_weights, topk_ids);
|
||||
}
|
||||
|
||||
// hash topk for DeepSeek V4 (expert IDs from precomputed lookup table)
|
||||
std::tuple<at::Tensor, at::Tensor> hash_topk_cpu(
|
||||
at::Tensor& gating_output,
|
||||
at::Tensor& tid2eid,
|
||||
int64_t topk,
|
||||
std::string scoring_func,
|
||||
int64_t num_fused_shared_experts,
|
||||
int64_t num_experts,
|
||||
double routed_scaling_factor) {
|
||||
CHECK_INPUT(gating_output);
|
||||
CHECK_INPUT(tid2eid);
|
||||
|
||||
const auto st = gating_output.scalar_type();
|
||||
int64_t num_tokens = gating_output.size(0);
|
||||
int64_t num_experts_gating = gating_output.size(1);
|
||||
int64_t routed_topk = topk - num_fused_shared_experts;
|
||||
|
||||
TORCH_CHECK(tid2eid.size(0) == num_tokens, "tid2eid row count must match num_tokens");
|
||||
TORCH_CHECK(tid2eid.size(1) == routed_topk, "tid2eid column count must match routed_topk");
|
||||
TORCH_CHECK(tid2eid.scalar_type() == at::kInt, "tid2eid must be int32");
|
||||
TORCH_CHECK(num_experts_gating == num_experts, "num_experts mismatch");
|
||||
|
||||
int scoring_func_id = 0;
|
||||
if (scoring_func == "softmax") {
|
||||
scoring_func_id = 0;
|
||||
} else if (scoring_func == "sigmoid") {
|
||||
scoring_func_id = 1;
|
||||
} else if (scoring_func == "sqrtsoftplus") {
|
||||
scoring_func_id = 2;
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported scoring_func: ", scoring_func);
|
||||
}
|
||||
|
||||
float routed_scaling_factor_value = static_cast<float>(routed_scaling_factor);
|
||||
|
||||
at::Tensor topk_weights = at::empty({num_tokens, topk}, gating_output.options().dtype(at::kFloat));
|
||||
at::Tensor topk_ids = at::empty({num_tokens, topk}, gating_output.options().dtype(at::kInt));
|
||||
|
||||
// tid2eid is [num_tokens, routed_topk], already indexed by input_ids in
|
||||
// Python; CHECK_INPUT above already guarantees it is contiguous.
|
||||
const at::Tensor& tid2eid_flat = tid2eid;
|
||||
|
||||
// Dispatch for bf16, fp16, and float32
|
||||
// Note: cannot use AT_DISPATCH_FLOATING_TYPES_AND2 since it includes double which lacks convert_to_float
|
||||
auto dispatch_fn = [&]<typename scalar_t>() {
|
||||
// For DeepSeek V4: 256 experts, routed_topk in {6,7,8} depending on
|
||||
// layer (num_fused_shared_experts is always 0 on this CPU call path --
|
||||
// see cpu_moe.py). Anything outside this table isn't a real model
|
||||
// shape, so fail loudly instead of silently degrading to a slow
|
||||
// generic path.
|
||||
switch (num_experts) {
|
||||
case 64:
|
||||
switch (routed_topk) {
|
||||
case 6:
|
||||
LAUNCH_HASH_TOPK_KERNEL(64, 6);
|
||||
break;
|
||||
case 7:
|
||||
LAUNCH_HASH_TOPK_KERNEL(64, 7);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_HASH_TOPK_KERNEL(64, 8);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "hash_topk_cpu: unsupported routed_topk ", routed_topk,
|
||||
" for num_experts=64");
|
||||
}
|
||||
break;
|
||||
case 128:
|
||||
switch (routed_topk) {
|
||||
case 6:
|
||||
LAUNCH_HASH_TOPK_KERNEL(128, 6);
|
||||
break;
|
||||
case 7:
|
||||
LAUNCH_HASH_TOPK_KERNEL(128, 7);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_HASH_TOPK_KERNEL(128, 8);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "hash_topk_cpu: unsupported routed_topk ", routed_topk,
|
||||
" for num_experts=128");
|
||||
}
|
||||
break;
|
||||
case 256:
|
||||
switch (routed_topk) {
|
||||
case 6:
|
||||
LAUNCH_HASH_TOPK_KERNEL(256, 6);
|
||||
break;
|
||||
case 7:
|
||||
LAUNCH_HASH_TOPK_KERNEL(256, 7);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_HASH_TOPK_KERNEL(256, 8);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "hash_topk_cpu: unsupported routed_topk ", routed_topk,
|
||||
" for num_experts=256");
|
||||
}
|
||||
break;
|
||||
case 384:
|
||||
switch (routed_topk) {
|
||||
case 6:
|
||||
LAUNCH_HASH_TOPK_KERNEL(384, 6);
|
||||
break;
|
||||
case 7:
|
||||
LAUNCH_HASH_TOPK_KERNEL(384, 7);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_HASH_TOPK_KERNEL(384, 8);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "hash_topk_cpu: unsupported routed_topk ", routed_topk,
|
||||
" for num_experts=384");
|
||||
}
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "hash_topk_cpu: unsupported num_experts ", num_experts);
|
||||
}
|
||||
};
|
||||
|
||||
if (st == at::ScalarType::BFloat16) {
|
||||
dispatch_fn.template operator()<at::BFloat16>();
|
||||
} else if (st == at::ScalarType::Half) {
|
||||
dispatch_fn.template operator()<at::Half>();
|
||||
} else if (st == at::ScalarType::Float) {
|
||||
dispatch_fn.template operator()<float>();
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported dtype for hash_topk_cpu: ", st);
|
||||
}
|
||||
|
||||
return std::make_tuple(topk_weights, topk_ids);
|
||||
}
|
||||
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu:
|
||||
// `topk_transform_512_cpu`/`topk_transform_512_cpu_kernel_impl`. Near-
|
||||
// verbatim port for the sparse indexer's DECODE path: hardcodes top-512
|
||||
// (the only `index_topk` DeepSeek-V4-Flash uses) and streams a min-heap over
|
||||
// the tail of the row instead of sorting the whole row.
|
||||
//
|
||||
// Differences from upstream:
|
||||
// - `out_page_indices` is still computed (required by the kernel contract)
|
||||
// but vLLM's caller only consumes `out_raw_indices` (local/compressed-
|
||||
// context positions -- `DeepseekV4CPUAttention.forward_mqa` resolves those
|
||||
// to physical slots itself via `map_local_to_global_slots_cpu`).
|
||||
// - `page_size` was already a runtime argument upstream, unlike
|
||||
// `fp8_paged_mqa_logits_cpu` (paged_mqa_logits.cpp), which needed one.
|
||||
namespace {
|
||||
|
||||
constexpr int64_t kC4Topk = 512;
|
||||
|
||||
template <typename T>
|
||||
inline int64_t load_index_value(const T* __restrict__ ptr, int64_t idx) {
|
||||
return static_cast<int64_t>(ptr[idx]);
|
||||
}
|
||||
|
||||
struct TopKTransformElem {
|
||||
float score;
|
||||
int32_t index;
|
||||
};
|
||||
|
||||
struct TopKTransformMinHeapCmp {
|
||||
bool operator()(const TopKTransformElem& lhs, const TopKTransformElem& rhs) const {
|
||||
if (lhs.score == rhs.score) {
|
||||
return lhs.index > rhs.index;
|
||||
}
|
||||
return lhs.score > rhs.score;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename seq_t, typename page_t>
|
||||
void topk_transform_512_cpu_kernel_impl(
|
||||
const float* __restrict__ scores,
|
||||
const seq_t* __restrict__ seq_lens,
|
||||
const page_t* __restrict__ page_tables,
|
||||
int32_t* __restrict__ out_page_indices,
|
||||
int32_t* __restrict__ out_raw_indices,
|
||||
int64_t batch_size,
|
||||
int64_t max_seq_len,
|
||||
int64_t page_table_stride,
|
||||
int64_t out_stride,
|
||||
int64_t page_size) {
|
||||
TORCH_CHECK(page_size > 0, "page_size must be positive");
|
||||
TORCH_CHECK((page_size & (page_size - 1)) == 0, "page_size must be a power of 2");
|
||||
const int page_bits = page_size > 1 ? static_cast<int>(std::log2(static_cast<double>(page_size))) : 0;
|
||||
const int64_t page_mask = page_size - 1;
|
||||
|
||||
at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) {
|
||||
std::array<TopKTransformElem, kC4Topk> heap;
|
||||
|
||||
for (int64_t b = begin; b < end; ++b) {
|
||||
const float* __restrict__ scores_row = scores + b * max_seq_len;
|
||||
const page_t* __restrict__ page_table_row = page_tables + b * page_table_stride;
|
||||
int32_t* __restrict__ out_page_row = out_page_indices + b * out_stride;
|
||||
int32_t* __restrict__ out_raw_row = out_raw_indices == nullptr ? nullptr : out_raw_indices + b * out_stride;
|
||||
|
||||
int64_t seq_len = load_index_value(seq_lens, b);
|
||||
seq_len = std::max<int64_t>(0, std::min<int64_t>(seq_len, max_seq_len));
|
||||
const int64_t valid_topk = std::min<int64_t>(seq_len, kC4Topk);
|
||||
|
||||
auto store_slot = [&](int64_t slot, int32_t raw_index) {
|
||||
if (raw_index < 0) {
|
||||
out_page_row[slot] = -1;
|
||||
if (out_raw_row != nullptr) {
|
||||
out_raw_row[slot] = -1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const int64_t page_idx = static_cast<int64_t>(raw_index) >> page_bits;
|
||||
const int64_t offset_in_page = static_cast<int64_t>(raw_index) & page_mask;
|
||||
const int64_t physical_page = load_index_value(page_table_row, page_idx);
|
||||
out_page_row[slot] = static_cast<int32_t>((physical_page << page_bits) | offset_in_page);
|
||||
if (out_raw_row != nullptr) {
|
||||
out_raw_row[slot] = raw_index;
|
||||
}
|
||||
};
|
||||
|
||||
if (seq_len <= kC4Topk) {
|
||||
for (int64_t i = 0; i < valid_topk; ++i) {
|
||||
store_slot(i, static_cast<int32_t>(i));
|
||||
}
|
||||
for (int64_t i = valid_topk; i < kC4Topk; ++i) {
|
||||
store_slot(i, -1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
for (int64_t i = 0; i < kC4Topk; ++i) {
|
||||
heap[i] = {scores_row[i], static_cast<int32_t>(i)};
|
||||
}
|
||||
std::make_heap(heap.begin(), heap.end(), TopKTransformMinHeapCmp());
|
||||
|
||||
for (int64_t i = kC4Topk; i < seq_len; ++i) {
|
||||
const float score = scores_row[i];
|
||||
const TopKTransformElem& current_min = heap.front();
|
||||
if (score > current_min.score ||
|
||||
(score == current_min.score && static_cast<int32_t>(i) < current_min.index)) {
|
||||
std::pop_heap(heap.begin(), heap.end(), TopKTransformMinHeapCmp());
|
||||
heap.back() = {score, static_cast<int32_t>(i)};
|
||||
std::push_heap(heap.begin(), heap.end(), TopKTransformMinHeapCmp());
|
||||
}
|
||||
}
|
||||
|
||||
for (int64_t i = 0; i < kC4Topk; ++i) {
|
||||
store_slot(i, heap[i].index);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void topk_transform_512_cpu(
|
||||
at::Tensor& scores,
|
||||
at::Tensor& seq_lens,
|
||||
at::Tensor& page_tables,
|
||||
at::Tensor& out_page_indices,
|
||||
int64_t page_size,
|
||||
const std::optional<at::Tensor>& out_raw_indices) {
|
||||
CHECK_INPUT(scores);
|
||||
CHECK_INPUT(seq_lens);
|
||||
CHECK_INPUT(page_tables);
|
||||
CHECK_INPUT(out_page_indices);
|
||||
|
||||
TORCH_CHECK(scores.scalar_type() == at::kFloat, "scores must be float32");
|
||||
TORCH_CHECK(out_page_indices.scalar_type() == at::kInt, "out_page_indices must be int32");
|
||||
TORCH_CHECK(scores.dim() == 2, "scores must be a 2D tensor");
|
||||
TORCH_CHECK(seq_lens.dim() == 1, "seq_lens must be a 1D tensor");
|
||||
TORCH_CHECK(page_tables.dim() == 2, "page_tables must be a 2D tensor");
|
||||
TORCH_CHECK(out_page_indices.dim() == 2, "out_page_indices must be a 2D tensor");
|
||||
|
||||
const int64_t batch_size = scores.size(0);
|
||||
const int64_t max_seq_len = scores.size(1);
|
||||
TORCH_CHECK(seq_lens.size(0) == batch_size, "seq_lens row count must match scores");
|
||||
TORCH_CHECK(page_tables.size(0) == batch_size, "page_tables row count must match scores");
|
||||
TORCH_CHECK(out_page_indices.size(0) == batch_size, "out_page_indices row count must match scores");
|
||||
TORCH_CHECK(out_page_indices.size(1) >= kC4Topk, "out_page_indices must have at least 512 columns");
|
||||
|
||||
int32_t* raw_ptr = nullptr;
|
||||
if (out_raw_indices.has_value()) {
|
||||
at::Tensor raw = out_raw_indices.value();
|
||||
CHECK_INPUT(raw);
|
||||
TORCH_CHECK(raw.scalar_type() == at::kInt, "out_raw_indices must be int32");
|
||||
TORCH_CHECK(raw.dim() == 2, "out_raw_indices must be a 2D tensor");
|
||||
TORCH_CHECK(raw.sizes() == out_page_indices.sizes(), "out_raw_indices shape must match out_page_indices");
|
||||
raw_ptr = raw.data_ptr<int32_t>();
|
||||
}
|
||||
|
||||
if (batch_size == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto launch_with_seq_type = [&](auto seq_zero) {
|
||||
using seq_t = decltype(seq_zero);
|
||||
if (page_tables.scalar_type() == at::kInt) {
|
||||
topk_transform_512_cpu_kernel_impl<seq_t, int32_t>(
|
||||
scores.data_ptr<float>(), seq_lens.data_ptr<seq_t>(), page_tables.data_ptr<int32_t>(),
|
||||
out_page_indices.data_ptr<int32_t>(), raw_ptr, batch_size, max_seq_len, page_tables.stride(0),
|
||||
out_page_indices.stride(0), page_size);
|
||||
} else if (page_tables.scalar_type() == at::kLong) {
|
||||
topk_transform_512_cpu_kernel_impl<seq_t, int64_t>(
|
||||
scores.data_ptr<float>(), seq_lens.data_ptr<seq_t>(), page_tables.data_ptr<int64_t>(),
|
||||
out_page_indices.data_ptr<int32_t>(), raw_ptr, batch_size, max_seq_len, page_tables.stride(0),
|
||||
out_page_indices.stride(0), page_size);
|
||||
} else {
|
||||
TORCH_CHECK(false, "page_tables must be int32 or int64");
|
||||
}
|
||||
};
|
||||
|
||||
if (seq_lens.scalar_type() == at::kInt) {
|
||||
launch_with_seq_type(int32_t{0});
|
||||
} else if (seq_lens.scalar_type() == at::kLong) {
|
||||
launch_with_seq_type(int64_t{0});
|
||||
} else {
|
||||
TORCH_CHECK(false, "seq_lens must be int32 or int64");
|
||||
}
|
||||
}
|
||||
+213
-16
@@ -66,18 +66,18 @@ at::Tensor convert_weight_packed(at::Tensor& weight);
|
||||
|
||||
at::Tensor convert_scale_packed(at::Tensor& scale);
|
||||
|
||||
at::Tensor fused_experts_cpu(
|
||||
at::Tensor& hidden_states, at::Tensor& w1, at::Tensor& w2,
|
||||
at::Tensor& topk_weights, at::Tensor& topk_ids, bool inplace,
|
||||
int64_t moe_comp_method, const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<at::Tensor>& w1_zero,
|
||||
const std::optional<at::Tensor>& w2_zero,
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
const std::optional<at::Tensor>& w1_bias,
|
||||
const std::optional<at::Tensor>& w2_bias,
|
||||
const std::optional<double>& alpha, const std::optional<double>& limit,
|
||||
bool is_vnni);
|
||||
void fused_experts_cpu(at::Tensor& out, at::Tensor& hidden_states,
|
||||
at::Tensor& w1, at::Tensor& w2, at::Tensor& topk_weights,
|
||||
at::Tensor& topk_ids, int64_t moe_comp_method,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<at::Tensor>& w1_zero,
|
||||
const std::optional<at::Tensor>& w2_zero,
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
const std::optional<at::Tensor>& w1_bias,
|
||||
const std::optional<at::Tensor>& w2_bias,
|
||||
const std::optional<double>& alpha,
|
||||
const std::optional<double>& limit, bool is_vnni);
|
||||
|
||||
at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2,
|
||||
at::Tensor& scales2,
|
||||
@@ -124,6 +124,99 @@ void concat_and_cache_mla_cpu(const at::Tensor& kv_c_normed,
|
||||
const at::Tensor& k_pe, at::Tensor& kv_cache,
|
||||
const at::Tensor& slot_mapping);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 mHC gating kernels
|
||||
std::tuple<at::Tensor, at::Tensor, at::Tensor> hc_pre_fused_cpu(
|
||||
at::Tensor& x, at::Tensor& hc_fn, at::Tensor& hc_scale, at::Tensor& hc_base,
|
||||
int64_t hc_mult, int64_t sinkhorn_iters, double rms_eps, double hc_eps);
|
||||
|
||||
at::Tensor hc_post_fused_cpu(at::Tensor& x, at::Tensor& residual,
|
||||
at::Tensor& post, at::Tensor& comb);
|
||||
|
||||
at::Tensor hc_head_fused_cpu(at::Tensor& x, at::Tensor& hc_fn,
|
||||
at::Tensor& hc_scale, at::Tensor& hc_base,
|
||||
double hc_eps, double norm_eps);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 fp8_ds_mla cache-write kernel
|
||||
at::Tensor fused_qnorm_rope_kv_insert_cpu(at::Tensor& q, at::Tensor& kv,
|
||||
at::Tensor& positions,
|
||||
at::Tensor& swa_kv_cache_2d,
|
||||
at::Tensor& slot_mapping,
|
||||
at::Tensor& cos_sin_cache,
|
||||
int64_t q_head_padded, double eps,
|
||||
int64_t cache_block_size);
|
||||
|
||||
// vLLM-native: DeepSeek-V4 sparse MQA attention kernel for the fp8_ds_mla
|
||||
// cache layout (SWA window + optional compressed top-k index sets).
|
||||
void flash_mla_with_kvcache_cpu(
|
||||
at::Tensor& out, at::Tensor& q, at::Tensor& window_cache_2d,
|
||||
at::Tensor& window_slots, int64_t window_block_size,
|
||||
at::Tensor& compressed_cache_2d, at::Tensor& compressed_slots,
|
||||
int64_t compressed_block_size, at::Tensor& attn_sink, double scale);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 compressor state-cache write + fused
|
||||
// compress+RMSNorm+quant+RoPE+store kernels.
|
||||
void save_partial_states_cpu(at::Tensor& kv, at::Tensor& score, at::Tensor& ape,
|
||||
at::Tensor& positions, at::Tensor& state_cache,
|
||||
at::Tensor& slot_mapping);
|
||||
|
||||
void compress_norm_rope_store_cpu(
|
||||
at::Tensor& state_cache, at::Tensor& gather_slots, at::Tensor& positions,
|
||||
at::Tensor& kv_slot_mapping, at::Tensor& rms_norm_weight,
|
||||
double rms_norm_eps, at::Tensor& cos_sin_cache, at::Tensor& kv_cache_2d,
|
||||
int64_t kv_cache_block_size, int64_t compress_ratio);
|
||||
|
||||
// head_dim=128 indexer-compressor variant of the above (single fp8 quant
|
||||
// block + raw fp32 scale -- matches the paged indexer K-cache layout
|
||||
// fp8_paged_mqa_logits_cpu/topk_transform_512_cpu read).
|
||||
void compress_norm_rope_store_indexer_cpu(
|
||||
at::Tensor& state_cache, at::Tensor& gather_slots, at::Tensor& positions,
|
||||
at::Tensor& kv_slot_mapping, at::Tensor& rms_norm_weight,
|
||||
double rms_norm_eps, at::Tensor& cos_sin_cache, at::Tensor& kv_cache_2d,
|
||||
int64_t kv_cache_block_size, int64_t compress_ratio);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 sparse indexer paged MQA-logits +
|
||||
// top-512 transform kernels (DECODE path -- reads the paged K-cache
|
||||
// directly via `page_table`, whole batch in one call).
|
||||
at::Tensor fp8_paged_mqa_logits_cpu(at::Tensor& q_fp8, at::Tensor& kvcache_fp8,
|
||||
at::Tensor& weight, at::Tensor& seq_lens,
|
||||
at::Tensor& page_table, int64_t block_size,
|
||||
int64_t max_seq_len);
|
||||
|
||||
void topk_transform_512_cpu(at::Tensor& scores, at::Tensor& seq_lens,
|
||||
at::Tensor& page_tables,
|
||||
at::Tensor& out_page_indices, int64_t page_size,
|
||||
const std::optional<at::Tensor>& out_raw_indices);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 MoE routing kernels (flat biased top-k and
|
||||
// hash-routed-layer top-k).
|
||||
std::tuple<at::Tensor, at::Tensor> biased_topk_cpu(
|
||||
at::Tensor& hidden_states, at::Tensor& gating_output,
|
||||
at::Tensor& correction_bias, int64_t topk, bool renormalize,
|
||||
std::string scoring_func, int64_t num_fused_shared_experts,
|
||||
std::optional<double> routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output);
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor> hash_topk_cpu(
|
||||
at::Tensor& gating_output, at::Tensor& tid2eid, int64_t topk,
|
||||
std::string scoring_func, int64_t num_fused_shared_experts,
|
||||
int64_t num_experts, double routed_scaling_factor);
|
||||
|
||||
// vLLM-native: DeepSeek-V4 sparse indexer Q-side RoPE + FP8 quant kernel
|
||||
// (fp8 path only; MXFP4 stays on triton-cpu).
|
||||
void fused_indexer_q_rope_quant_cpu(at::Tensor& positions, at::Tensor& index_q,
|
||||
at::Tensor& index_q_cos_sin_cache,
|
||||
at::Tensor& index_q_fp8,
|
||||
at::Tensor& index_weights,
|
||||
double index_weights_softmax_scale,
|
||||
double index_weights_head_scale,
|
||||
at::Tensor& index_weights_out);
|
||||
|
||||
// vLLM-native: inverse GPT-J RoPE for DeepSeek-V4 CPU attention's `_o_proj`
|
||||
// output de-rotation.
|
||||
at::Tensor inverse_gptj_rope_o_proj_cpu(at::Tensor& o, at::Tensor& positions,
|
||||
at::Tensor& cos_sin_cache,
|
||||
int64_t rope_dim);
|
||||
|
||||
// Adapted from sglang: INT4 W4A8 kernels
|
||||
std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_weight_packed_scale_zp(
|
||||
at::Tensor qweight, // awq: (*, K, N / 8) || gptq: (*, K / 8, N) , int32
|
||||
@@ -515,13 +608,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
ops.def("convert_scale_packed(Tensor! scale) -> Tensor");
|
||||
ops.impl("convert_scale_packed", torch::kCPU, &convert_scale_packed);
|
||||
ops.def(
|
||||
"fused_experts_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor "
|
||||
"topk_weights, Tensor topk_ids, bool "
|
||||
"inplace, int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
|
||||
"fused_experts_cpu(Tensor(a0!) out, Tensor hidden_states, Tensor w1, "
|
||||
"Tensor w2, Tensor topk_weights, Tensor topk_ids, "
|
||||
"int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
|
||||
"Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, "
|
||||
"Tensor? w1_bias, Tensor? w2_bias, float? alpha, float? limit, "
|
||||
"bool is_vnni) -> "
|
||||
"Tensor");
|
||||
"()");
|
||||
ops.impl("fused_experts_cpu", torch::kCPU, &fused_experts_cpu);
|
||||
ops.def(
|
||||
"int8_scaled_mm_with_quant(Tensor mat1, Tensor mat2, Tensor scales2, "
|
||||
@@ -583,6 +676,110 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"concat_and_cache_mla_cpu(Tensor kv_c_normed, Tensor k_pe, "
|
||||
"Tensor(a!) kv_cache, Tensor slot_mapping) -> ()");
|
||||
ops.impl("concat_and_cache_mla_cpu", torch::kCPU, &concat_and_cache_mla_cpu);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 mHC gating kernels
|
||||
ops.def(
|
||||
"hc_pre_fused_cpu(Tensor x, Tensor hc_fn, Tensor hc_scale, "
|
||||
"Tensor hc_base, int hc_mult, int sinkhorn_iters, float rms_eps, "
|
||||
"float hc_eps) -> (Tensor, Tensor, Tensor)");
|
||||
ops.impl("hc_pre_fused_cpu", torch::kCPU, &hc_pre_fused_cpu);
|
||||
|
||||
ops.def(
|
||||
"hc_post_fused_cpu(Tensor x, Tensor residual, Tensor post, "
|
||||
"Tensor comb) -> Tensor");
|
||||
ops.impl("hc_post_fused_cpu", torch::kCPU, &hc_post_fused_cpu);
|
||||
|
||||
ops.def(
|
||||
"hc_head_fused_cpu(Tensor x, Tensor hc_fn, Tensor "
|
||||
"hc_scale, Tensor hc_base, float hc_eps, float norm_eps) -> "
|
||||
"Tensor");
|
||||
ops.impl("hc_head_fused_cpu", torch::kCPU, &hc_head_fused_cpu);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 fp8_ds_mla cache-write kernel
|
||||
ops.def(
|
||||
"fused_qnorm_rope_kv_insert_cpu(Tensor q, Tensor kv, Tensor positions, "
|
||||
"Tensor(a!) swa_kv_cache_2d, Tensor slot_mapping, Tensor "
|
||||
"cos_sin_cache, int q_head_padded, float eps, int cache_block_size) "
|
||||
"-> Tensor");
|
||||
ops.impl("fused_qnorm_rope_kv_insert_cpu", torch::kCPU,
|
||||
&fused_qnorm_rope_kv_insert_cpu);
|
||||
|
||||
// vLLM-native: DeepSeek-V4 sparse MQA attention kernel
|
||||
ops.def(
|
||||
"flash_mla_with_kvcache_cpu(Tensor(a!) out, Tensor q, Tensor "
|
||||
"window_cache_2d, Tensor window_slots, int window_block_size, Tensor "
|
||||
"compressed_cache_2d, Tensor compressed_slots, int "
|
||||
"compressed_block_size, Tensor attn_sink, float scale) -> ()");
|
||||
ops.impl("flash_mla_with_kvcache_cpu", torch::kCPU,
|
||||
&flash_mla_with_kvcache_cpu);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 compressor kernels
|
||||
ops.def(
|
||||
"save_partial_states_cpu(Tensor kv, Tensor score, Tensor ape, "
|
||||
"Tensor positions, Tensor(a!) state_cache, Tensor slot_mapping) "
|
||||
"-> ()");
|
||||
ops.impl("save_partial_states_cpu", torch::kCPU, &save_partial_states_cpu);
|
||||
|
||||
ops.def(
|
||||
"compress_norm_rope_store_cpu(Tensor state_cache, Tensor "
|
||||
"gather_slots, Tensor positions, Tensor kv_slot_mapping, Tensor "
|
||||
"rms_norm_weight, float rms_norm_eps, Tensor cos_sin_cache, "
|
||||
"Tensor(a!) kv_cache_2d, int kv_cache_block_size, int compress_ratio) "
|
||||
"-> ()");
|
||||
ops.impl("compress_norm_rope_store_cpu", torch::kCPU,
|
||||
&compress_norm_rope_store_cpu);
|
||||
|
||||
ops.def(
|
||||
"compress_norm_rope_store_indexer_cpu(Tensor state_cache, Tensor "
|
||||
"gather_slots, Tensor positions, Tensor kv_slot_mapping, Tensor "
|
||||
"rms_norm_weight, float rms_norm_eps, Tensor cos_sin_cache, "
|
||||
"Tensor(a!) kv_cache_2d, int kv_cache_block_size, int compress_ratio) "
|
||||
"-> ()");
|
||||
ops.impl("compress_norm_rope_store_indexer_cpu", torch::kCPU,
|
||||
&compress_norm_rope_store_indexer_cpu);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 sparse indexer paged MQA-logits kernel
|
||||
// (DECODE path)
|
||||
ops.def(
|
||||
"fp8_paged_mqa_logits_cpu(Tensor q_fp8, Tensor kvcache_fp8, Tensor "
|
||||
"weight, Tensor seq_lens, Tensor page_table, int block_size, int "
|
||||
"max_seq_len) -> Tensor");
|
||||
ops.impl("fp8_paged_mqa_logits_cpu", torch::kCPU, &fp8_paged_mqa_logits_cpu);
|
||||
|
||||
ops.def(
|
||||
"topk_transform_512_cpu(Tensor scores, Tensor seq_lens, Tensor "
|
||||
"page_tables, Tensor(a!) out_page_indices, int page_size, Tensor(b!)? "
|
||||
"out_raw_indices) -> ()");
|
||||
ops.impl("topk_transform_512_cpu", torch::kCPU, &topk_transform_512_cpu);
|
||||
|
||||
ops.def(
|
||||
"fused_indexer_q_rope_quant_cpu(Tensor positions, Tensor index_q, "
|
||||
"Tensor index_q_cos_sin_cache, Tensor(a!) index_q_fp8, Tensor "
|
||||
"index_weights, float index_weights_softmax_scale, float "
|
||||
"index_weights_head_scale, Tensor(b!) index_weights_out) -> ()");
|
||||
ops.impl("fused_indexer_q_rope_quant_cpu", torch::kCPU,
|
||||
&fused_indexer_q_rope_quant_cpu);
|
||||
|
||||
// Adapted from sglang: DeepSeek-V4 MoE routing kernels
|
||||
ops.def(
|
||||
"biased_topk_cpu(Tensor hidden_states, Tensor gating_output, Tensor "
|
||||
"correction_bias, int topk, bool renormalize, str scoring_func, int "
|
||||
"num_fused_shared_experts, float? routed_scaling_factor, bool "
|
||||
"apply_routed_scaling_factor_on_output) -> (Tensor, Tensor)");
|
||||
ops.impl("biased_topk_cpu", torch::kCPU, &biased_topk_cpu);
|
||||
|
||||
ops.def(
|
||||
"hash_topk_cpu(Tensor gating_output, Tensor tid2eid, int topk, str "
|
||||
"scoring_func, int num_fused_shared_experts, int num_experts, float "
|
||||
"routed_scaling_factor) -> (Tensor, Tensor)");
|
||||
ops.impl("hash_topk_cpu", torch::kCPU, &hash_topk_cpu);
|
||||
|
||||
// vLLM-native: inverse GPT-J RoPE for _o_proj's output de-rotation
|
||||
ops.def(
|
||||
"inverse_gptj_rope_o_proj_cpu(Tensor o, Tensor positions, Tensor "
|
||||
"cos_sin_cache, int rope_dim) -> Tensor");
|
||||
ops.impl("inverse_gptj_rope_o_proj_cpu", torch::kCPU,
|
||||
&inverse_gptj_rope_o_proj_cpu);
|
||||
#endif
|
||||
|
||||
#if (defined(__AVX512BF16__) && defined(__AVX512F__) && \
|
||||
|
||||
Reference in New Issue
Block a user