[CPU] Add CPU FP8 W8A8 linear/MoE support (#49942)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
6df2b1a8c6
commit
b346479065
+2
-11
@@ -1,16 +1,7 @@
|
||||
#include "cpu_attn_dispatch_generated.h"
|
||||
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#include <cpuid.h>
|
||||
// Runtime check for AMX-FP8 (Diamond Rapids): CPUID leaf 7, subleaf 1, EAX[21].
|
||||
static bool runtime_has_amx_fp8() {
|
||||
unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
|
||||
if (__get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx)) {
|
||||
return (eax >> 21) & 1u;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
#endif // x86_64
|
||||
// Runtime check for AMX-FP8, implemented in cpu_isa.cpp.
|
||||
extern bool runtime_has_amx_fp8();
|
||||
|
||||
// Maps kv_cache_dtype string to Fp8KVCacheDataType enum.
|
||||
// "auto" -> kAuto(0); "fp8"/"fp8_e4m3" -> kFp8E4M3; "fp8_e5m2" -> kFp8E5M2.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
// AMX-FP8 is reported at CPUID leaf 0x1E, subleaf 1, EAX[4].
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#include <cpuid.h>
|
||||
|
||||
bool runtime_has_amx_fp8() {
|
||||
unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
|
||||
if (__get_cpuid_max(0, nullptr) < 0x1e) return false;
|
||||
if (__get_cpuid_count(0x1e, 1, &eax, &ebx, &ecx, &edx)) {
|
||||
return (eax >> 4) & 1u;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool cpu_has_amx_fp8() { return runtime_has_amx_fp8(); }
|
||||
#else
|
||||
bool cpu_has_amx_fp8() { return false; }
|
||||
#endif
|
||||
@@ -12,11 +12,45 @@
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/Dispatch.h>
|
||||
#include <ATen/Parallel.h>
|
||||
#include <cstring>
|
||||
|
||||
#if defined(_OPENMP)
|
||||
#include <omp.h>
|
||||
#endif
|
||||
|
||||
// Adapted from sglang: dispatch utilities for FP8 W8A8 kernels
|
||||
#define AT_DISPATCH_OUT_TYPES(TYPE, NAME, ...) \
|
||||
AT_DISPATCH_SWITCH( \
|
||||
TYPE, \
|
||||
NAME, \
|
||||
AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Float, out_t, __VA_ARGS__) \
|
||||
AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::BFloat16, out_t, __VA_ARGS__) \
|
||||
AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Half, out_t, __VA_ARGS__))
|
||||
|
||||
#define AT_DISPATCH_CASE_ENUM_NO_RETURN(VALUE, TYPE, HINT, ...) \
|
||||
case VALUE: { \
|
||||
constexpr TYPE HINT = VALUE; \
|
||||
__VA_ARGS__; \
|
||||
break; \
|
||||
}
|
||||
|
||||
#define AT_DISPATCH_BOOL_NO_RETURN(VALUE, NAME, HINT, ...) \
|
||||
[&]() { \
|
||||
switch (VALUE) { \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(true, bool, HINT, __VA_ARGS__) \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(false, bool, HINT, __VA_ARGS__) \
|
||||
} \
|
||||
}()
|
||||
|
||||
#define AT_DISPATCH_QUANT_MODE_NO_RETURN(MODE, NAME, HINT, ...) \
|
||||
[&]() { \
|
||||
switch (MODE) { \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_TENSOR, int, HINT, __VA_ARGS__) \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_ROW, int, HINT, __VA_ARGS__) \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_GROUP, int, HINT, __VA_ARGS__) \
|
||||
} \
|
||||
}()
|
||||
|
||||
namespace {
|
||||
|
||||
// dispatch bool
|
||||
@@ -472,4 +506,53 @@ inline T* conditional_data_ptr(const std::optional<at::Tensor>& opt) {
|
||||
return opt.has_value() ? opt.value().data_ptr<T>() : nullptr;
|
||||
}
|
||||
|
||||
// Adapted from sglang: M blocking utilities for FP8 W8A8 kernels
|
||||
inline int64_t get_m_block(int64_t M) {
|
||||
if (M <= 48) {
|
||||
return M;
|
||||
} else if (M < 64) {
|
||||
return 32;
|
||||
} else if (M < 96) {
|
||||
return 64;
|
||||
} else {
|
||||
return 128;
|
||||
}
|
||||
}
|
||||
|
||||
inline std::tuple<bool, int64_t, int64_t, int64_t> get_m_blocking(int64_t M) {
|
||||
bool parallel_on_M = M > 128;
|
||||
int64_t block_m = get_m_block(M);
|
||||
int64_t Mc = (M + block_m - 1) / block_m;
|
||||
int64_t Mc_parallel = parallel_on_M ? Mc : 1;
|
||||
return std::make_tuple(parallel_on_M, block_m, Mc, Mc_parallel);
|
||||
}
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
template <typename T>
|
||||
void zero_buffer(T* data, int64_t size) {
|
||||
const int32_t vec_size = at::vec::Vectorized<T>::size();
|
||||
auto zero_vec = at::vec::Vectorized<T>(0);
|
||||
int64_t d = 0;
|
||||
for (; d < size - (size % vec_size); d += vec_size) {
|
||||
zero_vec.store(data + d);
|
||||
}
|
||||
if (d < size) {
|
||||
zero_vec.store(data + d, size - d);
|
||||
}
|
||||
}
|
||||
#else
|
||||
template <typename T>
|
||||
void zero_buffer(T* data, int64_t size) {
|
||||
memset(data, 0, sizeof(T) * size);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__x86_64__)
|
||||
bool avx10_2_available() {
|
||||
// __builtin_cpu_supports returns the masked feature bit rather than 0/1.
|
||||
static const bool available = __builtin_cpu_supports("avx10.2");
|
||||
return available;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
@@ -94,7 +94,7 @@ enum class CPUActMethod : int {
|
||||
gelu_and_mul = 2,
|
||||
};
|
||||
|
||||
enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4 };
|
||||
enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4, FP8_W8A8 = 5 };
|
||||
|
||||
constexpr bool operator==(CPUQuantMethod a, int64_t b) {
|
||||
return static_cast<int64_t>(a) == b;
|
||||
@@ -395,3 +395,55 @@ void tinygemm_kernel(
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack = true);
|
||||
|
||||
// FP8 W8A8 tinygemm kernel (defined in gemm_fp8_w8a8.cpp)
|
||||
void tinygemm_kernel(
|
||||
float* C,
|
||||
const at::Float8_e4m3fn* A,
|
||||
const float* scales_a,
|
||||
const at::Float8_e4m3fn* B,
|
||||
const float* scales_b,
|
||||
int64_t M,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldc,
|
||||
int64_t ldsa,
|
||||
float* ukernel_buf,
|
||||
at::BFloat16* dqA_buf,
|
||||
at::BFloat16* dqB_buf);
|
||||
|
||||
// FP8 W8A8 quantize activation (defined in gemm_fp8_w8a8.cpp)
|
||||
std::tuple<at::Tensor, at::Tensor> quantize_fp8e4m3_vec(
|
||||
const at::Tensor& t,
|
||||
bool channelwise,
|
||||
c10::optional<at::Tensor> scale_opt);
|
||||
|
||||
// FP8 W8A8 fused MoE kernel (defined in moe_fp8_w8a8.cpp)
|
||||
template <typename scalar_t>
|
||||
void fused_experts_fp8_a8_kernel_impl(
|
||||
scalar_t* __restrict__ output,
|
||||
scalar_t* __restrict__ ic0,
|
||||
scalar_t* __restrict__ ic1,
|
||||
scalar_t* __restrict__ ic2,
|
||||
at::Float8_e4m3fn* __restrict__ A_tmp,
|
||||
scalar_t* __restrict__ B_tmp,
|
||||
float* __restrict__ C_tmp,
|
||||
float* __restrict__ Ukernel_tmp,
|
||||
const at::Float8_e4m3fn* __restrict__ input,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w1,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w2,
|
||||
const float* __restrict__ As,
|
||||
const float* __restrict__ w1s,
|
||||
const float* __restrict__ w2s,
|
||||
int64_t block_size_N,
|
||||
int64_t block_size_K,
|
||||
const float* __restrict__ topk_weights,
|
||||
const int32_t* __restrict__ sorted_ids,
|
||||
const int32_t* __restrict__ expert_ids,
|
||||
const int32_t* __restrict__ offsets,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -836,6 +836,11 @@ static inline void check_moe_scales(
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for mxfp4.");
|
||||
TORCH_CHECK(w1_scale.value().scalar_type() == at::kByte, "expect w1_scale to be uint8.");
|
||||
TORCH_CHECK(w2_scale.value().scalar_type() == at::kByte, "expect w2_scale to be uint8.");
|
||||
} else if constexpr (quant == CPUQuantMethod::FP8_W8A8) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a8.");
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for fp8 w8a8.");
|
||||
TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a8.");
|
||||
TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -850,9 +855,14 @@ static inline void check_moe_scales(
|
||||
check_moe_scales<CPUQuantMethod::FP8_W8A16>(w1_scale, w2_scale, block_size);
|
||||
} else if (moe_comp_method == CPUQuantMethod::MXFP4) {
|
||||
check_moe_scales<CPUQuantMethod::MXFP4>(w1_scale, w2_scale, block_size);
|
||||
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
|
||||
check_moe_scales<CPUQuantMethod::FP8_W8A8>(w1_scale, w2_scale, block_size);
|
||||
}
|
||||
}
|
||||
|
||||
// CHECK_MOE_SCALES_FP8: validate scale shapes for FP8_W8A16.
|
||||
// w1s shape [E, 2N/block_N, K/block_K, block_N]: DIM0=1, DIM1=2
|
||||
// w2s shape [E, K/block_N, N/block_K]: DIM0=1, DIM1=2
|
||||
#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \
|
||||
auto w1s = w1_scale.value(); \
|
||||
auto w2s = w2_scale.value(); \
|
||||
@@ -864,6 +874,17 @@ static inline void check_moe_scales(
|
||||
TORCH_CHECK(w2s.size(DIM0) == div_up(K, block_size_N)); \
|
||||
TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K))
|
||||
|
||||
// CHECK_MOE_SCALES_FP8_W8A8: extract scale info for FP8_W8A8 without shape
|
||||
// validation, because w1s is pre-packed by float8_linear_prepack_cpu with
|
||||
// shape [E, 2N/BLOCK_N, G, BLOCK_N] where BLOCK_N=32 != block_size_N(128).
|
||||
// This matches the sglang upstream behavior (checks are commented out there).
|
||||
#define CHECK_MOE_SCALES_FP8_W8A8(DIM0, DIM1) \
|
||||
auto w1s = w1_scale.value(); \
|
||||
auto w2s = w2_scale.value(); \
|
||||
auto block_size_val = block_size.value(); \
|
||||
int64_t block_size_N = block_size_val[0]; \
|
||||
int64_t block_size_K = block_size_val[1]
|
||||
|
||||
// hidden_states: [M, K]
|
||||
// w1: [E, 2N, K] or [E, 2N, K / 2] for uint8
|
||||
// w2: [E, K, N] or [E, K, N / 2] for uint8
|
||||
@@ -883,6 +904,7 @@ void fused_experts_cpu(
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<at::Tensor>& w1_zero,
|
||||
const std::optional<at::Tensor>& w2_zero,
|
||||
const std::optional<at::Tensor>& a1_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
const std::optional<at::Tensor>& w1_bias,
|
||||
const std::optional<at::Tensor>& w2_bias,
|
||||
@@ -895,7 +917,10 @@ void fused_experts_cpu(
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
|
||||
const auto st = hidden_states.scalar_type();
|
||||
// FP8_W8A8 hidden_states input is FP8; scalar_t should be BF16 for output
|
||||
const auto st = (moe_comp_method == CPUQuantMethod::FP8_W8A8)
|
||||
? at::kBFloat16
|
||||
: hidden_states.scalar_type();
|
||||
// TODO: fused_topk_torch_native (CPU fallback for models like MiniMax)
|
||||
// returns int64 topk_ids; fused_experts_cpu requires int32. Remove the typecast after topk kernel is provided
|
||||
auto topk_ids_ = topk_ids.scalar_type() == at::kInt ? topk_ids : topk_ids.to(at::kInt);
|
||||
@@ -908,6 +933,9 @@ void fused_experts_cpu(
|
||||
if (moe_comp_method == CPUQuantMethod::INT4_W4A8 && is_vnni) {
|
||||
CHECK_DIM(4, w1);
|
||||
CHECK_DIM(4, w2);
|
||||
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
|
||||
// w1 is prepackaged by float8_linear_prepack_cpu (5D), skip w1 dim check
|
||||
CHECK_DIM(3, w2);
|
||||
} else {
|
||||
CHECK_DIM(3, w1);
|
||||
CHECK_DIM(3, w2);
|
||||
@@ -924,8 +952,9 @@ void fused_experts_cpu(
|
||||
|
||||
int64_t M = hidden_states.size(0);
|
||||
int64_t K = hidden_states.size(1);
|
||||
int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8 ? w1_scale.value().size(1) * w1_scale.value().size(3) / 2
|
||||
: w1.size(1) / 2;
|
||||
int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8
|
||||
? w1_scale.value().size(1) * w1_scale.value().size(3) / 2
|
||||
: (moe_comp_method == CPUQuantMethod::FP8_W8A8 ? w2.size(2) : w1.size(1) / 2);
|
||||
int64_t E = w1.size(0);
|
||||
int64_t topk = topk_weights_.size(1);
|
||||
|
||||
@@ -935,7 +964,10 @@ void fused_experts_cpu(
|
||||
|
||||
// check weight shapes
|
||||
CHECK_EQ(w2.size(0), E);
|
||||
if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) {
|
||||
if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
|
||||
CHECK_EQ(w2.size(1), K);
|
||||
CHECK_EQ(packed_w2.size(2), packed_N);
|
||||
} else if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) {
|
||||
CHECK_EQ(w2.size(1), K);
|
||||
CHECK_EQ(packed_w1.size(2), packed_K / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1));
|
||||
CHECK_EQ(packed_w2.size(2), packed_N / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1));
|
||||
@@ -993,22 +1025,35 @@ void fused_experts_cpu(
|
||||
// 5. Aq_tmp : [M, K] or [M * topk, N]
|
||||
// 6. As_tmp : [M * topk]
|
||||
//
|
||||
// for fp8 w8a16 and mxfp4:
|
||||
// for fp8 w8a16, mxfp4, and fp8 w8a8:
|
||||
// 7. intermediate_cache0 : [M * topk, 2N]
|
||||
// 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)]
|
||||
//
|
||||
// for fp8 w8a8 additionally:
|
||||
// 9. Ukernel_tmp : [T, 2 * BLOCK_M * BLOCK_N] (float32 for FP8×FP8 brgemm)
|
||||
//
|
||||
int64_t buffer_size_nbytes =
|
||||
M * topk * N * 2 + M * topk * K * 2 +
|
||||
num_threads * BLOCK_M * K *
|
||||
(moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 1 : 2) +
|
||||
(moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8
|
||||
| moe_comp_method == CPUQuantMethod::FP8_W8A8 ? 1 : 2) +
|
||||
num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float);
|
||||
|
||||
if (moe_comp_method == CPUQuantMethod::INT8_W8A8) {
|
||||
buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float);
|
||||
}
|
||||
if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4) {
|
||||
if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4
|
||||
|| moe_comp_method == CPUQuantMethod::FP8_W8A8) {
|
||||
buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2;
|
||||
}
|
||||
if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
|
||||
// Ukernel buffer: float32 workspace per thread for FP8×FP8 brgemm
|
||||
buffer_size_nbytes += num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float);
|
||||
#ifndef CPUBLAS_BRGEMM_F8F8F32
|
||||
// BF16 fallback: also need dqA [T, BLOCK_M, BLOCK_K] + dqB [T, BLOCK_K, BLOCK_N]
|
||||
buffer_size_nbytes += num_threads * (BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N) * 2;
|
||||
#endif
|
||||
}
|
||||
if (moe_comp_method == CPUQuantMethod::INT4_W4A8) {
|
||||
buffer_size_nbytes += M * topk * 2 * N * 2 + std::max(M * K, M * topk * N) + M * topk * sizeof(float) +
|
||||
num_threads * 2 * get_4bit_block_k_size(K / w1_scale.value().size(2)) * BLOCK_N;
|
||||
@@ -1181,6 +1226,47 @@ void fused_experts_cpu(
|
||||
E,
|
||||
topk,
|
||||
num_tokens_post_pad);
|
||||
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
|
||||
// Stage 1: FP8×FP8 GEMM (hidden_states is FP8 input)
|
||||
// Stage 2: BF16×FP8 GEMM (W8A16 path for down-projection)
|
||||
at::Float8_e4m3fn* __restrict__ A_tmp =
|
||||
(at::Float8_e4m3fn*)((void*)(intermediate_cache2 + M * topk * K));
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
scalar_t* __restrict__ intermediate_cache0 =
|
||||
(scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
|
||||
scalar_t* __restrict__ B_tmp =
|
||||
(scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
|
||||
float* __restrict__ Ukernel_tmp =
|
||||
(float*)((void*)(B_tmp + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N)));
|
||||
|
||||
CHECK_MOE_SCALES_FP8_W8A8(1, 2);
|
||||
fused_experts_fp8_a8_kernel_impl(
|
||||
out_hidden_states.data_ptr<scalar_t>(),
|
||||
intermediate_cache0,
|
||||
intermediate_cache1,
|
||||
intermediate_cache2,
|
||||
A_tmp,
|
||||
B_tmp,
|
||||
C_tmp,
|
||||
Ukernel_tmp,
|
||||
hidden_states.data_ptr<at::Float8_e4m3fn>(),
|
||||
packed_w1.data_ptr<at::Float8_e4m3fn>(),
|
||||
packed_w2.data_ptr<at::Float8_e4m3fn>(),
|
||||
a1_scale.has_value() ? a1_scale.value().data_ptr<float>() : nullptr,
|
||||
w1s.data_ptr<float>(),
|
||||
w2s.data_ptr<float>(),
|
||||
block_size_N,
|
||||
block_size_K,
|
||||
topk_weights_.data_ptr<float>(),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
offsets,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
E,
|
||||
topk,
|
||||
num_tokens_post_pad);
|
||||
} else {
|
||||
scalar_t* __restrict__ A_tmp = intermediate_cache2 + M * topk * K;
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
|
||||
@@ -0,0 +1,432 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
// clang-format off
|
||||
|
||||
#include <c10/util/Unroll.h>
|
||||
|
||||
#include "common.h"
|
||||
#include "gemm.h"
|
||||
#include "moe.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// Byte-wise copy for FP8 elements (no remainder assumption)
|
||||
inline void copy_stub2(
|
||||
at::Float8_e4m3fn* __restrict__ out,
|
||||
const at::Float8_e4m3fn* __restrict__ input,
|
||||
int64_t size) {
|
||||
for (int64_t d = 0; d < size; d++) {
|
||||
out[d] = input[d];
|
||||
}
|
||||
}
|
||||
|
||||
#define PER_TENSOR 1
|
||||
#define PER_ROW 2
|
||||
#define PER_GROUP 3
|
||||
|
||||
// Store float32 GEMM result to output buffer with dequantization and dtype
|
||||
// conversion. Activations are PER_ROW quantized; weights are PER_GROUP quantized.
|
||||
// Uses AVX512 on capable platforms.
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
|
||||
template <typename out_dtype, int64_t N, int act_quant_mode, int wei_quant_mode>
|
||||
inline void store_out(
|
||||
const float* y_buf,
|
||||
out_dtype* c_ptr,
|
||||
int64_t M,
|
||||
int64_t lda,
|
||||
const float* scales_a,
|
||||
const float* scales_b,
|
||||
const float* bias) {
|
||||
float a_scale = 1.0, b_scale = 1.0;
|
||||
__m512 va_scale, vb_scale;
|
||||
if constexpr (act_quant_mode == PER_TENSOR) {
|
||||
a_scale = *scales_a;
|
||||
}
|
||||
if constexpr (wei_quant_mode == PER_TENSOR) {
|
||||
b_scale = *scales_b;
|
||||
vb_scale = _mm512_set1_ps(b_scale);
|
||||
}
|
||||
for (int i = 0; i < M; ++i) {
|
||||
if constexpr (act_quant_mode == PER_ROW) {
|
||||
a_scale = *(scales_a + i);
|
||||
}
|
||||
if constexpr (act_quant_mode != PER_GROUP) {
|
||||
va_scale = _mm512_set1_ps(a_scale);
|
||||
}
|
||||
constexpr int N_UNROLL = N / 16;
|
||||
c10::ForcedUnroll<N_UNROLL>{}([&](auto idx) {
|
||||
constexpr int j = idx * 16;
|
||||
__m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j);
|
||||
__m512 bias_vec = bias ? _mm512_loadu_ps(bias + j) : _mm512_setzero_ps();
|
||||
if constexpr (act_quant_mode != PER_GROUP) {
|
||||
y_vec = _mm512_mul_ps(y_vec, va_scale);
|
||||
}
|
||||
if constexpr (wei_quant_mode == PER_ROW) {
|
||||
vb_scale = _mm512_loadu_ps(scales_b + j);
|
||||
}
|
||||
if constexpr (wei_quant_mode != PER_GROUP) {
|
||||
y_vec = _mm512_mul_ps(y_vec, vb_scale);
|
||||
}
|
||||
y_vec = _mm512_add_ps(y_vec, bias_vec);
|
||||
if constexpr (std::is_same<out_dtype, float>::value) {
|
||||
_mm512_storeu_ps(c_ptr + i * lda + j, y_vec);
|
||||
} else if constexpr (std::is_same<out_dtype, at::BFloat16>::value) {
|
||||
__m256i y_bf16_vec = at::vec::cvtfp32_bf16(y_vec);
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_bf16_vec);
|
||||
} else if constexpr (std::is_same<out_dtype, at::Half>::value) {
|
||||
__m256i y_fp16_vec = at::vec::cvtfp32_fp16(y_vec);
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_fp16_vec);
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported output dtype");
|
||||
}
|
||||
});
|
||||
constexpr int tail_start = N / 16 * 16;
|
||||
for (int j = tail_start; j < N; ++j) {
|
||||
if constexpr (wei_quant_mode == PER_ROW) {
|
||||
b_scale = scales_b[j];
|
||||
}
|
||||
c_ptr[i * lda + j] = static_cast<out_dtype>(y_buf[i * N + j] * a_scale * b_scale);
|
||||
}
|
||||
} // for M
|
||||
}
|
||||
|
||||
#else // no AVX512
|
||||
|
||||
template <typename out_dtype, int64_t N, int act_quant_mode, int wei_quant_mode>
|
||||
inline void store_out(
|
||||
const float* y_buf,
|
||||
out_dtype* c_ptr,
|
||||
int64_t M,
|
||||
int64_t lda,
|
||||
const float* scales_a,
|
||||
const float* scales_b,
|
||||
const float* bias) {
|
||||
float a_scale = 1.0, b_scale = 1.0;
|
||||
if constexpr (act_quant_mode == PER_TENSOR) {
|
||||
a_scale = *scales_a;
|
||||
}
|
||||
if constexpr (wei_quant_mode == PER_TENSOR) {
|
||||
b_scale = *scales_b;
|
||||
}
|
||||
for (int i = 0; i < M; ++i) {
|
||||
if constexpr (act_quant_mode == PER_ROW) {
|
||||
a_scale = *(scales_a + i);
|
||||
}
|
||||
for (int j = 0; j < N; ++j) {
|
||||
if constexpr (wei_quant_mode == PER_ROW) {
|
||||
b_scale = scales_b[j];
|
||||
}
|
||||
float val = y_buf[i * N + j] * a_scale * b_scale;
|
||||
if (bias) val += bias[j];
|
||||
c_ptr[i * lda + j] = static_cast<out_dtype>(val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif // CPU_CAPABILITY_AVX512
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
// FP8 W8A8 fused MoE kernel.
|
||||
//
|
||||
// hidden_states: [M, K] FP8 E4M3, pre-quantized activation
|
||||
// packed_w1: [E, 2*Nc, Kc, BLOCK_K, BLOCK_N] FP8 W8A8 prepack (float8_linear_prepack_cpu)
|
||||
// packed_w2: [E, K, N_packed] FP8 W8A16 VNNI pack (convert_weight_packed)
|
||||
// As: [M] float32 per-token activation scales
|
||||
// w1s: [E, num_groups, 2N] float32 per-group weight scales for w1
|
||||
// w2s: [E, ...] float32 per-group weight scales for w2
|
||||
//
|
||||
// Stage 1: hidden_states(FP8) @ w1(FP8) → ic0(BF16) [M*topk, 2N]
|
||||
// Uses FP8×FP8 GEMM (tinygemm_kernel with FP8 inputs)
|
||||
// Stage 1.5: ic1 = silu(ic0[:,:N]) * ic0[:,N:] [M*topk, N]
|
||||
// Stage 2: ic1(BF16) @ w2(FP8) → ic2(BF16) [M*topk, K]
|
||||
// Uses BF16×FP8 GEMM (W8A16 path, tinygemm_kernel<scalar_t>)
|
||||
// Stage 3: out = weighted_sum(ic2, topk_weights) [M, K]
|
||||
|
||||
template <typename scalar_t>
|
||||
void fused_experts_fp8_a8_kernel_impl(
|
||||
scalar_t* __restrict__ output,
|
||||
scalar_t* __restrict__ ic0,
|
||||
scalar_t* __restrict__ ic1,
|
||||
scalar_t* __restrict__ ic2,
|
||||
at::Float8_e4m3fn* __restrict__ A_tmp,
|
||||
scalar_t* __restrict__ B_tmp,
|
||||
float* __restrict__ C_tmp,
|
||||
float* __restrict__ Ukernel_tmp,
|
||||
const at::Float8_e4m3fn* __restrict__ input,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w1,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w2,
|
||||
const float* __restrict__ As,
|
||||
const float* __restrict__ w1s,
|
||||
const float* __restrict__ w2s,
|
||||
int64_t block_size_N,
|
||||
int64_t block_size_K,
|
||||
const float* __restrict__ topk_weights,
|
||||
const int32_t* __restrict__ sorted_ids,
|
||||
const int32_t* __restrict__ expert_ids,
|
||||
const int32_t* __restrict__ offsets,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad) {
|
||||
// 1. intermediate_cache1 : [M * topk, N]
|
||||
// 2. intermediate_cache2 : [M * topk, K]
|
||||
// 3. A_tmp : [T, BLOCK_M * K] FP8
|
||||
// 4. C_tmp : [T, 2 * BLOCK_M * BLOCK_N] float32
|
||||
// 5. intermediate_cache0 : [M * topk, 2N]
|
||||
// 6. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, max(K, N)] scalar_t
|
||||
// 7. Ukernel_tmp: [T, 2 * BLOCK_M * BLOCK_N] float32
|
||||
// 8. (non-FP8-brgemm only) dqA [T, BLOCK_M * BLOCK_K] + dqB [T, BLOCK_K * BLOCK_N] in BF16
|
||||
// appended after Ukernel_tmp; see moe.cpp FP8_W8A8 buffer allocation
|
||||
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N_VAL = block_size_n(); // = 32
|
||||
const int64_t KB = K / BLOCK_K; // K blocks
|
||||
|
||||
int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N_VAL * std::max(K, N);
|
||||
|
||||
// stage 1: intermediate_cache0 = hidden_states @ w1
|
||||
const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M);
|
||||
const int64_t NB = div_up(N, BLOCK_N_VAL);
|
||||
|
||||
int64_t scale_size_N = div_up(2 * N, block_size_N);
|
||||
int64_t scale_size_K = div_up(K, block_size_K);
|
||||
int64_t blocks_n_per_group = block_size_N / BLOCK_N_VAL;
|
||||
int64_t num_groups = div_up(K, block_size_K); // G
|
||||
int64_t blocks_k_per_group = block_size_K / BLOCK_K;
|
||||
const int64_t stride_e = 2 * N * K; // per-expert stride for w1 (total elements)
|
||||
bool use_brgemm = true;
|
||||
int64_t num_thread = at::get_num_threads();
|
||||
|
||||
// For the BF16 fallback path (no native FP8 brgemm), tinygemm_kernel needs temporary
|
||||
// BF16 buffers to dequantize FP8 inputs before running AMX-BF16 brgemm.
|
||||
// These are placed after the Ukernel_tmp region (see moe.cpp FP8_W8A8 buffer allocation).
|
||||
#ifndef CPUBLAS_BRGEMM_F8F8F32
|
||||
// dqA: [T, BLOCK_M, BLOCK_K] BF16
|
||||
// dqB: [T, BLOCK_K, BLOCK_N] BF16
|
||||
// Layout: immediately after Ukernel_tmp[num_thread * 2 * BLOCK_M * BLOCK_N_VAL]
|
||||
at::BFloat16* __restrict__ dq_base = reinterpret_cast<at::BFloat16*>(
|
||||
Ukernel_tmp + num_thread * 2 * BLOCK_M * BLOCK_N_VAL);
|
||||
const int64_t dq_per_thread = BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N_VAL;
|
||||
#endif
|
||||
|
||||
at::parallel_for(0, MB * NB, 1, [&](int64_t begin, int64_t end) {
|
||||
int tid = get_thread_num();
|
||||
// A_tmp: FP8 activation buffer for this thread [BLOCK_M, K]
|
||||
at::Float8_e4m3fn* __restrict__ A = A_tmp + tid * BLOCK_M * K;
|
||||
float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL;
|
||||
float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N_VAL;
|
||||
alignas(64) float As_[BLOCK_M];
|
||||
float* __restrict__ ukernel_buf_1 = Ukernel_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL;
|
||||
#ifndef CPUBLAS_BRGEMM_F8F8F32
|
||||
at::BFloat16* __restrict__ dqA_buf = dq_base + tid * dq_per_thread;
|
||||
at::BFloat16* __restrict__ dqB_buf = dqA_buf + BLOCK_M * BLOCK_K;
|
||||
#else
|
||||
at::BFloat16* __restrict__ dqA_buf = nullptr;
|
||||
at::BFloat16* __restrict__ dqB_buf = nullptr;
|
||||
#endif
|
||||
float* __restrict__ ukernel_buf_2 = ukernel_buf_1 + BLOCK_M * BLOCK_N_VAL;
|
||||
auto ldsa = 1; // activation per-row quantized
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB;
|
||||
int64_t nb = i % NB;
|
||||
int64_t nb1 = nb + NB; // gate is first N columns, up is second N columns
|
||||
|
||||
int64_t n_size = std::min(N - nb * BLOCK_N_VAL, BLOCK_N_VAL);
|
||||
|
||||
// B (weight) for expert expert_id: gate+up fused [2N, K] prepackaged as FP8
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e;
|
||||
const float* __restrict__ Bs = w1s + expert_id * num_groups * (2 * N);
|
||||
|
||||
// Load activation tile for this M-block
|
||||
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
int32_t index = A_ids[m] / topk;
|
||||
copy_stub2(A + m * K, input + index * K, K);
|
||||
As_[m] = As[index];
|
||||
}
|
||||
|
||||
const int64_t offset = offsets[mb];
|
||||
|
||||
// Gate path (nb): w1[:N, :]
|
||||
zero_buffer(C0, BLOCK_M * BLOCK_N_VAL);
|
||||
for (int kci = 0; kci < KB; ++kci) {
|
||||
// FP8×FP8 tinygemm: act_quant=PER_ROW(2), wei_quant=PER_GROUP(3)
|
||||
tinygemm_kernel(
|
||||
/* C */ C0,
|
||||
/* A */ A + kci * BLOCK_K,
|
||||
/* scales_a*/ As_ + kci / blocks_k_per_group,
|
||||
/* B */ B + (nb * KB + kci) * BLOCK_K * BLOCK_N_VAL,
|
||||
/* scales_b*/ Bs + nb * BLOCK_N_VAL * num_groups + kci / blocks_k_per_group * BLOCK_N_VAL,
|
||||
/* M */ m_size,
|
||||
/* K */ BLOCK_K,
|
||||
/* lda */ K,
|
||||
/* ldc */ BLOCK_N_VAL,
|
||||
/* ldsa */ ldsa,
|
||||
/* ukernel */ ukernel_buf_1,
|
||||
/* dqA_buf */ dqA_buf,
|
||||
/* dqB_buf */ dqB_buf);
|
||||
}
|
||||
store_out<scalar_t, BLOCK_N_VAL, PER_ROW, PER_GROUP>(
|
||||
C0,
|
||||
ic0 + offset * 2 * N + nb * BLOCK_N_VAL,
|
||||
m_size,
|
||||
2 * N, // lda
|
||||
As_,
|
||||
nullptr,
|
||||
nullptr);
|
||||
|
||||
// Up path (nb1): w1[N:, :]
|
||||
zero_buffer(C1, BLOCK_M * BLOCK_N_VAL);
|
||||
for (int kci = 0; kci < KB; ++kci) {
|
||||
tinygemm_kernel(
|
||||
/* C */ C1,
|
||||
/* A */ A + kci * BLOCK_K,
|
||||
/* scales_a*/ As_ + kci / blocks_k_per_group,
|
||||
/* B */ B + (nb1 * KB + kci) * BLOCK_K * BLOCK_N_VAL,
|
||||
/* scales_b*/ Bs + nb1 * BLOCK_N_VAL * num_groups + kci / blocks_k_per_group * BLOCK_N_VAL,
|
||||
/* M */ m_size,
|
||||
/* K */ BLOCK_K,
|
||||
/* lda */ K,
|
||||
/* ldc */ BLOCK_N_VAL,
|
||||
/* ldsa */ ldsa,
|
||||
/* ukernel */ ukernel_buf_2,
|
||||
/* dqA_buf */ dqA_buf,
|
||||
/* dqB_buf */ dqB_buf);
|
||||
}
|
||||
store_out<scalar_t, BLOCK_N_VAL, PER_ROW, PER_GROUP>(
|
||||
C1,
|
||||
ic0 + offset * 2 * N + nb1 * BLOCK_N_VAL,
|
||||
m_size,
|
||||
2 * N, // lda
|
||||
As_,
|
||||
nullptr,
|
||||
nullptr);
|
||||
}
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
|
||||
// stage 1.5: intermediate_cache1 = silu(ic0[:, :N]) * ic0[:, N:]
|
||||
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N);
|
||||
}
|
||||
});
|
||||
|
||||
// stage 2: intermediate_cache2 = intermediate_cache1 @ w2
|
||||
// w2: [E, K, N] FP8, VNNI packed; treated as W8A16 (BF16 act × FP8 weight)
|
||||
const int64_t OC = K; // output channels (hidden dim)
|
||||
const int64_t IC = N; // input channels (expert intermediate dim)
|
||||
const int64_t MB2 = MB;
|
||||
const int64_t NB2 = div_up(OC, BLOCK_N_VAL);
|
||||
scale_size_N = div_up(K, block_size_N);
|
||||
scale_size_K = div_up(N, block_size_K);
|
||||
const int64_t stride_e2 = OC * IC; // per-expert stride for w2
|
||||
const int64_t stride_oc = IC;
|
||||
|
||||
int64_t avg_M = std::max(int64_t(1), M * topk / E);
|
||||
bool use_brgemm2 = can_use_brgemm<at::Float8_e4m3fn>(avg_M);
|
||||
|
||||
parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
int tid = get_thread_num();
|
||||
alignas(64) scalar_t C[BLOCK_M * BLOCK_K];
|
||||
|
||||
loop_2d<at::Float8_e4m3fn>(mb0, mb1, nb0, nb1, BLOCK_N_VAL * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
int64_t n_size = std::min(OC - nb * BLOCK_N_VAL, BLOCK_N_VAL);
|
||||
|
||||
// A from ic1: [M*topk, N] in sorted order (contiguous after stage 1.5)
|
||||
const scalar_t* __restrict__ A2 = ic1 + offsets[mb] * N;
|
||||
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
|
||||
|
||||
// B from w2: [E, K, N] VNNI packed, offset to expert + output block
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const at::Float8_e4m3fn* __restrict__ B2 =
|
||||
packed_w2 + expert_id * stride_e2 + nb * BLOCK_N_VAL * stride_oc;
|
||||
const float* __restrict__ Bs2 =
|
||||
w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K;
|
||||
|
||||
// Only unpack B on first visit or new expert
|
||||
int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1];
|
||||
bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id);
|
||||
|
||||
tinygemm_kernel<scalar_t>(
|
||||
/* A */ A2,
|
||||
/* B */ B2,
|
||||
/* C */ C,
|
||||
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N_VAL * IC,
|
||||
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL,
|
||||
/* Bbias */ nullptr,
|
||||
/* scale */ Bs2,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
/* K */ IC,
|
||||
/* lda */ IC,
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ BLOCK_N_VAL,
|
||||
/* brg */ use_brgemm2,
|
||||
/* block_sz_K */ block_size_K,
|
||||
/* do_unpack */ do_unpack);
|
||||
|
||||
// Scatter ic2 in original token order, weighting by topk_weights
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
int32_t index = A_ids[m];
|
||||
float weight = topk_weights[index];
|
||||
copy_mul_stub(ic2 + index * K + nb * BLOCK_N_VAL, C + m * BLOCK_N_VAL, weight, n_size);
|
||||
}
|
||||
});
|
||||
|
||||
if (use_brgemm2) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
|
||||
// stage 3: output = sum(ic2, dim=topk) [M, K]
|
||||
at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
sum_stub(output + m * K, ic2 + m * topk * K, topk, K);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#define INSTANTIATE_MOE_FP8_A8_TEMPLATE(TYPE) \
|
||||
template void fused_experts_fp8_a8_kernel_impl<TYPE>( \
|
||||
TYPE* __restrict__ output, \
|
||||
TYPE* __restrict__ ic0, \
|
||||
TYPE* __restrict__ ic1, \
|
||||
TYPE* __restrict__ ic2, \
|
||||
at::Float8_e4m3fn* __restrict__ A_tmp, \
|
||||
TYPE* __restrict__ B_tmp, \
|
||||
float* __restrict__ C_tmp, \
|
||||
float* __restrict__ Ukernel_tmp, \
|
||||
const at::Float8_e4m3fn* __restrict__ input, \
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w1, \
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w2, \
|
||||
const float* __restrict__ As, \
|
||||
const float* __restrict__ w1s, \
|
||||
const float* __restrict__ w2s, \
|
||||
int64_t block_size_N, \
|
||||
int64_t block_size_K, \
|
||||
const float* __restrict__ topk_weights, \
|
||||
const int32_t* __restrict__ sorted_ids, \
|
||||
const int32_t* __restrict__ expert_ids, \
|
||||
const int32_t* __restrict__ offsets, \
|
||||
int64_t M, \
|
||||
int64_t N, \
|
||||
int64_t K, \
|
||||
int64_t E, \
|
||||
int64_t topk, \
|
||||
int64_t num_tokens_post_pad)
|
||||
|
||||
INSTANTIATE_MOE_FP8_A8_TEMPLATE(at::BFloat16);
|
||||
INSTANTIATE_MOE_FP8_A8_TEMPLATE(at::Half);
|
||||
@@ -73,6 +73,7 @@ void fused_experts_cpu(at::Tensor& out, at::Tensor& hidden_states,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<at::Tensor>& w1_zero,
|
||||
const std::optional<at::Tensor>& w2_zero,
|
||||
const std::optional<at::Tensor>& a1_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
const std::optional<at::Tensor>& w1_bias,
|
||||
const std::optional<at::Tensor>& w2_bias,
|
||||
@@ -91,6 +92,18 @@ at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
at::ScalarType out_dtype, bool is_vnni);
|
||||
|
||||
// Adapted from sglang: FP8 W8A8 kernels
|
||||
std::tuple<at::Tensor, at::Tensor> float8_linear_prepack_impl(
|
||||
const at::Tensor& weight, const at::Tensor& scales);
|
||||
at::Tensor fp8_scaled_mm_with_quant(const at::Tensor& act,
|
||||
const std::optional<at::Tensor>& act_scales,
|
||||
bool channelwise, const at::Tensor& weight,
|
||||
const at::Tensor& weight_scales,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
at::ScalarType output_dtype);
|
||||
std::tuple<at::Tensor, at::Tensor> quantize_fp8e4m3_vec(
|
||||
const at::Tensor& t, bool channelwise, c10::optional<at::Tensor> scale_opt);
|
||||
|
||||
// Adapted from sglang: MLA CPU kernels (AMX-only)
|
||||
void decode_attention_cpu(at::Tensor& query, at::Tensor& k_buffer,
|
||||
at::Tensor& v_buffer, at::Tensor& output,
|
||||
@@ -281,6 +294,7 @@ void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
|
||||
const std::string& activation);
|
||||
|
||||
bool cpu_attn_has_isa(const std::string& isa);
|
||||
bool cpu_has_amx_fp8();
|
||||
|
||||
torch::Tensor get_scheduler_metadata(
|
||||
const int64_t num_req, const int64_t num_heads_q,
|
||||
@@ -611,7 +625,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"fused_experts_cpu(Tensor(a0!) out, Tensor hidden_states, Tensor w1, "
|
||||
"Tensor w2, Tensor topk_weights, Tensor topk_ids, "
|
||||
"int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
|
||||
"Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, "
|
||||
"Tensor? w1_zero, Tensor? w2_zero, Tensor? a1_scale, int[]? block_size, "
|
||||
"Tensor? w1_bias, Tensor? w2_bias, float? alpha, float? limit, "
|
||||
"bool is_vnni) -> "
|
||||
"()");
|
||||
@@ -629,7 +643,23 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"bool is_vnni) -> Tensor");
|
||||
ops.impl("fp8_scaled_mm_cpu", torch::kCPU, &fp8_scaled_mm_cpu);
|
||||
|
||||
// Adapted from sglang: casual_conv1d kernels
|
||||
// Adapted from sglang: FP8 W8A8 kernels
|
||||
ops.def(
|
||||
"float8_linear_prepack_cpu(Tensor weight, Tensor scales) -> (Tensor, "
|
||||
"Tensor)");
|
||||
ops.impl("float8_linear_prepack_cpu", torch::kCPU,
|
||||
&float8_linear_prepack_impl);
|
||||
ops.def(
|
||||
"fp8_scaled_mm_with_quant(Tensor act, Tensor? act_scales, bool "
|
||||
"channelwise, "
|
||||
"Tensor weight, Tensor weight_scales, Tensor? bias, ScalarType "
|
||||
"output_dtype) -> Tensor");
|
||||
ops.impl("fp8_scaled_mm_with_quant", torch::kCPU, &fp8_scaled_mm_with_quant);
|
||||
ops.def(
|
||||
"quantize_fp8e4m3_vec(Tensor input, bool channelwise, Tensor? scale_opt) "
|
||||
"-> (Tensor, Tensor)");
|
||||
ops.impl("quantize_fp8e4m3_vec", torch::kCPU, &quantize_fp8e4m3_vec);
|
||||
|
||||
ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor");
|
||||
ops.impl("causal_conv1d_weight_pack", torch::kCPU,
|
||||
&causal_conv1d_weight_pack);
|
||||
@@ -832,6 +862,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"-> (Tensor, Tensor)");
|
||||
ops.impl("fused_gdn_gating_cpu", torch::kCPU, &fused_gdn_gating_cpu);
|
||||
|
||||
// Pure runtime CPU ISA capability check.
|
||||
ops.def("cpu_has_amx_fp8() -> bool", &cpu_has_amx_fp8);
|
||||
|
||||
// CPU attention kernels
|
||||
ops.def("cpu_attn_has_isa(str isa) -> bool", &cpu_attn_has_isa);
|
||||
ops.def(
|
||||
|
||||
Reference in New Issue
Block a user