[CPU] Add MXFP4 W4A16 MoE support (#41922)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com> Signed-off-by: Yuwen Zhou <yuwen.zhou@intel.com>
This commit is contained in:
@@ -12,7 +12,7 @@ steps:
|
||||
- vllm/_custom_ops.py
|
||||
- tests/kernels/attention/test_cpu_attn.py
|
||||
- tests/kernels/moe/test_cpu_fused_moe.py
|
||||
- tests/kernels/moe/test_cpu_fp8_fused_moe.py
|
||||
- tests/kernels/moe/test_cpu_quant_fused_moe.py
|
||||
- tests/kernels/test_onednn.py
|
||||
- tests/kernels/test_awq_int4_to_int8.py
|
||||
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
|
||||
@@ -21,7 +21,7 @@ steps:
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
|
||||
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fp8_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/test_onednn.py
|
||||
pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
|
||||
pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py"
|
||||
@@ -70,7 +70,7 @@ steps:
|
||||
- tests/quantization/test_cpu_wna16.py
|
||||
commands:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m "
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
|
||||
pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
|
||||
pytest -x -v -s tests/quantization/test_cpu_wna16.py"
|
||||
|
||||
|
||||
@@ -348,6 +348,13 @@ inline int get_cache_blocks<at::Float8_e4m3fn>(int chunk_size) {
|
||||
return std::min(MAX_CACHE_BLOCK_SIZE, cache_block_size);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline int get_cache_blocks<uint8_t>(int chunk_size) {
|
||||
// mxfp4 uses bf16 as accumulate type
|
||||
int cache_block_size = get_cache_blocks<at::BFloat16>(chunk_size);
|
||||
return std::min(MAX_CACHE_BLOCK_SIZE, cache_block_size);
|
||||
}
|
||||
|
||||
// 2d sequential loop in range : [mb0, mb1), [nb0, nb1)
|
||||
template <typename T, typename func_t>
|
||||
inline void loop_2d(int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1, int64_t chunk_size, const func_t& f) {
|
||||
|
||||
@@ -72,7 +72,17 @@ inline int64_t get_row_size(int64_t K, bool use_int8_w8a8) {
|
||||
return use_int8_w8a8 ? K + sizeof(int32_t) : K;
|
||||
}
|
||||
|
||||
enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3 };
|
||||
enum class CPUAcTMethod : int { silu_and_mul = 0, swiglu = 1 };
|
||||
|
||||
constexpr bool operator==(CPUAcTMethod a, int b) {
|
||||
return static_cast<int>(a) == b;
|
||||
}
|
||||
|
||||
constexpr bool operator==(int a, CPUAcTMethod b) {
|
||||
return a == static_cast<int>(b);
|
||||
}
|
||||
|
||||
enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4 };
|
||||
|
||||
constexpr bool operator==(CPUQuantMethod a, int64_t b) {
|
||||
return static_cast<int64_t>(a) == b;
|
||||
@@ -99,6 +109,9 @@ inline int64_t get_4bit_block_k_size(int64_t group_size) {
|
||||
// pack weight to vnni format
|
||||
at::Tensor convert_weight_packed(at::Tensor& weight);
|
||||
|
||||
// pack scale to blocked format for mxfp4
|
||||
at::Tensor convert_scale_packed(at::Tensor& scale);
|
||||
|
||||
// pack weight to vnni format for int4
|
||||
std::tuple<at::Tensor, at::Tensor, at::Tensor>
|
||||
convert_weight_packed_scale_zp(at::Tensor qweight, at::Tensor qzeros, at::Tensor scales);
|
||||
@@ -129,9 +142,9 @@ void fused_experts_int8_kernel_impl(
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad);
|
||||
|
||||
// moe implementations for fp8 w8a16
|
||||
template <typename scalar_t>
|
||||
void fused_experts_fp8_kernel_impl(
|
||||
// moe implementations for fp8 w8a16 and mxfp4
|
||||
template <typename scalar_t, typename packed_t, typename param_t, bool is_mxfp4>
|
||||
void fused_experts_fp_kernel_impl(
|
||||
scalar_t* __restrict__ output,
|
||||
scalar_t* __restrict__ ic0,
|
||||
scalar_t* __restrict__ ic1,
|
||||
@@ -140,10 +153,12 @@ void fused_experts_fp8_kernel_impl(
|
||||
scalar_t* __restrict__ B_tmp,
|
||||
float* __restrict__ C_tmp,
|
||||
const scalar_t* __restrict__ input,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w1,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w2,
|
||||
const float* __restrict__ w1s,
|
||||
const float* __restrict__ w2s,
|
||||
const packed_t* __restrict__ packed_w1,
|
||||
const packed_t* __restrict__ packed_w2,
|
||||
const float* __restrict__ w1_bias,
|
||||
const float* __restrict__ w2_bias,
|
||||
const param_t* __restrict__ w1s,
|
||||
const param_t* __restrict__ w2s,
|
||||
int64_t block_size_N,
|
||||
int64_t block_size_K,
|
||||
const float* __restrict__ topk_weights,
|
||||
@@ -155,7 +170,11 @@ void fused_experts_fp8_kernel_impl(
|
||||
int64_t K,
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad);
|
||||
int64_t num_tokens_post_pad,
|
||||
float alpha,
|
||||
float limit,
|
||||
CPUAcTMethod act_func,
|
||||
bool with_bias);
|
||||
|
||||
// shared expert implementation for int8 w8a8
|
||||
template <typename scalar_t>
|
||||
@@ -266,6 +285,7 @@ void tinygemm_kernel(
|
||||
scalar_t* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
float* __restrict__ Ctmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const float* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
@@ -294,6 +314,26 @@ void tinygemm_kernel(
|
||||
int64_t ldc,
|
||||
bool brg);
|
||||
|
||||
// mxfp4
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
const uint8_t* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
float* __restrict__ Ctmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const uint8_t* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack = true);
|
||||
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
scalar_t* C,
|
||||
|
||||
@@ -67,6 +67,23 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__
|
||||
}
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void
|
||||
copy_add_stub(float* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) {
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = fVec::size();
|
||||
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data = fVec::loadu(input + d) + fVec::loadu(bias + d);
|
||||
data.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = input[d] + bias[d];
|
||||
}
|
||||
}
|
||||
|
||||
inline void unpack_B(
|
||||
at::BFloat16* __restrict__ Btmp,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_B,
|
||||
@@ -343,7 +360,6 @@ struct tinygemm_kernel_nn<at::BFloat16, at::Float8_e4m3fn, float, has_bias, BLOC
|
||||
Unroll<ROWS * COLS>{}(storec);
|
||||
}
|
||||
};
|
||||
|
||||
template <int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn2<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
static inline void apply(
|
||||
@@ -918,6 +934,7 @@ void tinygemm_kernel(
|
||||
scalar_t* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
float* __restrict__ Ctmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const float* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
@@ -928,6 +945,11 @@ void tinygemm_kernel(
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack) {
|
||||
if (Bbias != nullptr) {
|
||||
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, true>(
|
||||
A, B, C, Btmp, Ctmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
return;
|
||||
}
|
||||
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, false>(
|
||||
A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
}
|
||||
@@ -949,7 +971,6 @@ void tinygemm_kernel(
|
||||
bool brg) {
|
||||
tinygemm_kernel2<scalar_t>(A, B, C, Btmp, Ctmp, scale, M, N, K, lda, ldb, ldc, brg);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
@@ -957,6 +978,7 @@ void tinygemm_kernel(
|
||||
scalar_t* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
float* __restrict__ Ctmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const uint8_t* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
@@ -967,10 +989,68 @@ void tinygemm_kernel(
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack) {
|
||||
if (Bbias != nullptr) {
|
||||
tinygemm_kernel<scalar_t, uint8_t, uint8_t, true>(
|
||||
A, B, C, Btmp, Ctmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
return;
|
||||
}
|
||||
tinygemm_kernel<scalar_t, uint8_t, uint8_t, false>(
|
||||
A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
}
|
||||
|
||||
// tinygemm interface
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
const at::Float8_e4m3fn* __restrict__ B,
|
||||
float* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const float* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack) {
|
||||
if (Bbias != nullptr) {
|
||||
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, true>(
|
||||
A, B, C, Btmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
return;
|
||||
}
|
||||
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, false>(
|
||||
A, B, C, Btmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
const uint8_t* __restrict__ B,
|
||||
float* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const uint8_t* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack) {
|
||||
if (Bbias != nullptr) {
|
||||
tinygemm_kernel<scalar_t, uint8_t, uint8_t, true>(
|
||||
A, B, C, Btmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
return;
|
||||
}
|
||||
tinygemm_kernel<scalar_t, uint8_t, uint8_t, false>(
|
||||
A, B, C, Btmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
}
|
||||
|
||||
#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE_A, TYPE_B, TYPE_S) \
|
||||
template void tinygemm_kernel<TYPE_A>( \
|
||||
const TYPE_A* __restrict__ A, \
|
||||
@@ -978,6 +1058,7 @@ void tinygemm_kernel(
|
||||
TYPE_A* __restrict__ C, \
|
||||
TYPE_A* __restrict__ Btmp, \
|
||||
float* __restrict__ Ctmp, \
|
||||
const float* __restrict__ Bbias, \
|
||||
const TYPE_S* __restrict__ scale, \
|
||||
int64_t M, \
|
||||
int64_t N, \
|
||||
@@ -1020,7 +1101,6 @@ inline const float* get_bias_data(const std::optional<at::Tensor>& bias, int64_t
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// FP8 and MXFP4 WoQ uses the same pattern:
|
||||
// Btmp : [T, BLOCK_N * K]
|
||||
// Ctmp : [T, BLOCK_M * BLOCK_N]
|
||||
|
||||
@@ -812,6 +812,7 @@ void shared_expert_kernel_impl(
|
||||
static inline void check_moe_scales(
|
||||
bool use_int8_w8a8,
|
||||
bool use_fp8_w8a16,
|
||||
bool use_mxfp4,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size) {
|
||||
@@ -825,6 +826,12 @@ static inline void check_moe_scales(
|
||||
TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a16.");
|
||||
TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2.");
|
||||
}
|
||||
if (use_mxfp4) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for mxfp4.");
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for mxfp4.");
|
||||
TORCH_CHECK(w1_scale.value().scalar_type() == at::kByte, "expect w1_scale to be uint8.");
|
||||
TORCH_CHECK(w2_scale.value().scalar_type() == at::kByte, "expect w2_scale to be uint8.");
|
||||
}
|
||||
}
|
||||
|
||||
#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \
|
||||
@@ -839,8 +846,8 @@ static inline void check_moe_scales(
|
||||
TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K))
|
||||
|
||||
// hidden_states: [M, K]
|
||||
// w1: [E, 2N, K]
|
||||
// w2: [E, K, N]
|
||||
// w1: [E, 2N, K] or [E, 2N, K / 2] for uint8
|
||||
// w2: [E, K, N] or [E, K, N / 2] for uint8
|
||||
// topk_weights: [M, topk]
|
||||
// topk_ids: [M, topk] (int32_t)
|
||||
//
|
||||
@@ -858,6 +865,10 @@ at::Tensor fused_experts_cpu(
|
||||
const std::optional<at::Tensor>& w1_zero,
|
||||
const std::optional<at::Tensor>& w2_zero,
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
const std::optional<at::Tensor>& w1_bias,
|
||||
const std::optional<at::Tensor>& w2_bias,
|
||||
const std::optional<double>& alpha,
|
||||
const std::optional<double>& limit,
|
||||
bool is_vnni) {
|
||||
auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1);
|
||||
auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2);
|
||||
@@ -897,8 +908,12 @@ at::Tensor fused_experts_cpu(
|
||||
int64_t topk = topk_weights_.size(1);
|
||||
|
||||
// we use int32_t compensation for int8 w8a8
|
||||
int64_t packed_K = get_row_size(K, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
int64_t packed_N = get_row_size(N, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
int64_t packed_K = moe_comp_method == CPUQuantMethod::MXFP4
|
||||
? get_row_size<uint8_t>(K)
|
||||
: get_row_size(K, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
int64_t packed_N = moe_comp_method == CPUQuantMethod::MXFP4
|
||||
? get_row_size<uint8_t>(N)
|
||||
: get_row_size(N, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
|
||||
// check weight shapes
|
||||
CHECK_EQ(w2.size(0), E);
|
||||
@@ -911,6 +926,7 @@ at::Tensor fused_experts_cpu(
|
||||
check_moe_scales(
|
||||
moe_comp_method == CPUQuantMethod::INT8_W8A8,
|
||||
moe_comp_method == CPUQuantMethod::FP8_W8A16,
|
||||
moe_comp_method == CPUQuantMethod::MXFP4,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
block_size);
|
||||
@@ -965,7 +981,7 @@ at::Tensor fused_experts_cpu(
|
||||
// 5. Aq_tmp : [M, K] or [M * topk, N]
|
||||
// 6. As_tmp : [M * topk]
|
||||
//
|
||||
// for fp8 w8a16:
|
||||
// for fp8 w8a16 and mxfp4:
|
||||
// 7. intermediate_cache0 : [M * topk, 2N]
|
||||
// 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)]
|
||||
//
|
||||
@@ -978,7 +994,7 @@ at::Tensor fused_experts_cpu(
|
||||
if (moe_comp_method == CPUQuantMethod::INT8_W8A8) {
|
||||
buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float);
|
||||
}
|
||||
if (moe_comp_method == CPUQuantMethod::FP8_W8A16) {
|
||||
if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4) {
|
||||
buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2;
|
||||
}
|
||||
if (moe_comp_method == CPUQuantMethod::INT4_W4A8) {
|
||||
@@ -1031,9 +1047,11 @@ at::Tensor fused_experts_cpu(
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
|
||||
scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
|
||||
bool with_bias = w1_bias.has_value();
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul;
|
||||
|
||||
CHECK_MOE_SCALES_FP8(1, 2);
|
||||
fused_experts_fp8_kernel_impl(
|
||||
fused_experts_fp_kernel_impl<scalar_t, at::Float8_e4m3fn, float, false>(
|
||||
out_hidden_states.data_ptr<scalar_t>(),
|
||||
intermediate_cache0,
|
||||
intermediate_cache1,
|
||||
@@ -1044,6 +1062,8 @@ at::Tensor fused_experts_cpu(
|
||||
hidden_states.data_ptr<scalar_t>(),
|
||||
packed_w1.data_ptr<at::Float8_e4m3fn>(),
|
||||
packed_w2.data_ptr<at::Float8_e4m3fn>(),
|
||||
with_bias ? w1_bias.value().data_ptr<float>() : nullptr,
|
||||
with_bias ? w2_bias.value().data_ptr<float>() : nullptr,
|
||||
w1s.data_ptr<float>(),
|
||||
w2s.data_ptr<float>(),
|
||||
block_size_N,
|
||||
@@ -1057,7 +1077,56 @@ at::Tensor fused_experts_cpu(
|
||||
K,
|
||||
E,
|
||||
topk,
|
||||
num_tokens_post_pad);
|
||||
num_tokens_post_pad,
|
||||
alpha.has_value() ? float(alpha.value()) : 0,
|
||||
limit.has_value() ? float(limit.value()) : 0,
|
||||
act_func,
|
||||
with_bias);
|
||||
} else if (moe_comp_method == CPUQuantMethod::MXFP4) {
|
||||
scalar_t* __restrict__ A_tmp = (scalar_t*)((void*)(intermediate_cache2 + M * topk * K));
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
|
||||
scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
|
||||
bool with_bias = w1_bias.has_value();
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul;
|
||||
|
||||
// mxfp4 supports only group size of 32 (2^5)
|
||||
constexpr int64_t group_size = 32;
|
||||
auto w1s = w1_scale.value();
|
||||
auto w2s = w2_scale.value();
|
||||
TORCH_CHECK(w1s.numel() == E * 2 * N * K / group_size, "w1_scale size mismatch");
|
||||
TORCH_CHECK(w2s.numel() == E * K * N / group_size, "w2_scale size mismatch");
|
||||
fused_experts_fp_kernel_impl<scalar_t, uint8_t, uint8_t, true>(
|
||||
out_hidden_states.data_ptr<scalar_t>(),
|
||||
intermediate_cache0,
|
||||
intermediate_cache1,
|
||||
intermediate_cache2,
|
||||
A_tmp,
|
||||
B_tmp,
|
||||
C_tmp,
|
||||
hidden_states.data_ptr<scalar_t>(),
|
||||
packed_w1.data_ptr<uint8_t>(),
|
||||
packed_w2.data_ptr<uint8_t>(),
|
||||
with_bias ? w1_bias.value().data_ptr<float>() : nullptr,
|
||||
with_bias ? w2_bias.value().data_ptr<float>() : nullptr,
|
||||
w1s.data_ptr<uint8_t>(),
|
||||
w2s.data_ptr<uint8_t>(),
|
||||
/*block_size_N*/ 1,
|
||||
/*block_size_K*/ group_size,
|
||||
topk_weights_.data_ptr<float>(),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
offsets,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
E,
|
||||
topk,
|
||||
num_tokens_post_pad,
|
||||
alpha.has_value() ? float(alpha.value()) : 0,
|
||||
limit.has_value() ? float(limit.value()) : 0,
|
||||
act_func,
|
||||
with_bias);
|
||||
} else if (moe_comp_method == CPUQuantMethod::INT4_W4A8) {
|
||||
uint8_t* __restrict__ A_tmp = (uint8_t*)((void*)(intermediate_cache2 + M * topk * K));
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
@@ -1185,7 +1254,7 @@ at::Tensor shared_expert_cpu(
|
||||
CHECK_EQ(packed_w2.size(1), packed_N);
|
||||
|
||||
// check scales
|
||||
check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size);
|
||||
check_moe_scales(use_int8_w8a8, use_fp8_w8a16, false, w1_scale, w2_scale, block_size);
|
||||
|
||||
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
|
||||
|
||||
|
||||
@@ -176,3 +176,109 @@ inline void silu_and_mul_stub(
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
const fVec weight_vec = fVec(weight);
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d) * weight_vec;
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d] * weight);
|
||||
}
|
||||
}
|
||||
|
||||
// input = input + input2
|
||||
inline void add_bias_stub(float* __restrict__ input, const float* __restrict__ input2, int64_t size) {
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = fVec::size();
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec x_fvec = fVec::loadu(input + d);
|
||||
fVec y_fvec = fVec::loadu(input2 + d);
|
||||
x_fvec = x_fvec + y_fvec;
|
||||
x_fvec.store(input + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
input[d] = input[d] + input2[d];
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_mul_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, float weight, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
const fVec weight_vec = fVec(weight);
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
bVec x = bVec::loadu(input + d);
|
||||
fVec x0, x1;
|
||||
std::tie(x0, x1) = at::vec::convert_to_float(x);
|
||||
x0 = x0 * weight_vec;
|
||||
x1 = x1 * weight_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d] * weight);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void clamp_sigmoid_and_mul_stub(
|
||||
scalar_t* __restrict__ out,
|
||||
const scalar_t* __restrict__ input,
|
||||
int64_t size,
|
||||
const float alpha,
|
||||
const float limit) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
const fVec one = fVec(1.f);
|
||||
const fVec zero = fVec(0.f);
|
||||
const fVec limit_v = fVec(limit);
|
||||
const fVec nlimit_v = fVec(-limit);
|
||||
const fVec alpha_v = fVec(alpha);
|
||||
|
||||
// no remainder
|
||||
#pragma GCC unroll 4
|
||||
for (int64_t d = 0; d < size; d += bVec::size()) {
|
||||
bVec x = bVec::loadu(input + d);
|
||||
fVec x0_, y0_;
|
||||
std::tie(x0_, y0_) = at::vec::convert_to_float(x);
|
||||
float tmp_buffer[fVec::size() * 2]; // 32
|
||||
float tmp_glu[fVec::size()]; // 16
|
||||
float tmp_linear[fVec::size()]; // 16
|
||||
x0_.store(tmp_buffer);
|
||||
y0_.store(tmp_buffer + fVec::size());
|
||||
// interleaved: x[2i] = glu, x[2i+1] = linear
|
||||
for (int j = 0; j < fVec::size(); ++j) {
|
||||
// x0 [0,2,..30]
|
||||
tmp_glu[j] = tmp_buffer[j * 2];
|
||||
// y0 [1,3,...31]
|
||||
tmp_linear[j] = tmp_buffer[j * 2 + 1];
|
||||
}
|
||||
fVec x0 = fVec::loadu(tmp_glu);
|
||||
fVec y0 = fVec::loadu(tmp_linear);
|
||||
|
||||
// clamp
|
||||
x0 = at::vec::minimum(x0, limit_v);
|
||||
y0 = at::vec::minimum(limit_v, at::vec::maximum(nlimit_v, y0));
|
||||
// x * sigmoid(x * alpha)
|
||||
x0 = x0 / (one + (x0 * alpha_v).neg().exp_u20());
|
||||
// (y + 1) * x
|
||||
y0 = y0 + one;
|
||||
x0 = x0 * y0;
|
||||
convert_from_float_and_store<scalar_t>(out + d / 2, x0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
#include "gemm.h"
|
||||
#include "moe.h"
|
||||
|
||||
template <typename scalar_t>
|
||||
void fused_experts_fp8_kernel_impl(
|
||||
template <typename scalar_t, typename packed_t, typename param_t, bool is_mxfp4>
|
||||
void fused_experts_fp_kernel_impl(
|
||||
scalar_t* __restrict__ output,
|
||||
scalar_t* __restrict__ ic0,
|
||||
scalar_t* __restrict__ ic1,
|
||||
@@ -17,10 +17,12 @@ void fused_experts_fp8_kernel_impl(
|
||||
scalar_t* __restrict__ B_tmp,
|
||||
float* __restrict__ C_tmp,
|
||||
const scalar_t* __restrict__ input,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w1,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w2,
|
||||
const float* __restrict__ w1s,
|
||||
const float* __restrict__ w2s,
|
||||
const packed_t* __restrict__ packed_w1,
|
||||
const packed_t* __restrict__ packed_w2,
|
||||
const float* __restrict__ w1_bias,
|
||||
const float* __restrict__ w2_bias,
|
||||
const param_t* __restrict__ w1s,
|
||||
const param_t* __restrict__ w2s,
|
||||
int64_t block_size_N,
|
||||
int64_t block_size_K,
|
||||
const float* __restrict__ topk_weights,
|
||||
@@ -32,7 +34,11 @@ void fused_experts_fp8_kernel_impl(
|
||||
int64_t K,
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad) {
|
||||
int64_t num_tokens_post_pad,
|
||||
float alpha,
|
||||
float limit,
|
||||
CPUAcTMethod act_func,
|
||||
bool with_bias) {
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
|
||||
@@ -42,12 +48,20 @@ void fused_experts_fp8_kernel_impl(
|
||||
int64_t scale_size_N = div_up(2 * N, block_size_N);
|
||||
int64_t scale_size_K = div_up(K, block_size_K);
|
||||
int64_t blocks_n_per_group = block_size_N / BLOCK_N;
|
||||
std::function<int64_t(int64_t)> scale_offset_per_block;
|
||||
if constexpr (is_mxfp4) {
|
||||
scale_offset_per_block = [&](int64_t a) { return a * BLOCK_N; };
|
||||
} else {
|
||||
scale_offset_per_block = [&](int64_t a) { return a / blocks_n_per_group; };
|
||||
}
|
||||
|
||||
const int64_t stride_e = 2 * N * K;
|
||||
const int64_t stride_n = K;
|
||||
const int64_t packed_K = get_row_size<packed_t>(K);
|
||||
|
||||
const int64_t stride_e = 2 * N * packed_K;
|
||||
const int64_t stride_n = packed_K;
|
||||
|
||||
int64_t avg_M = std::max(int64_t(1), M * topk / E);
|
||||
const bool use_brgemm = can_use_brgemm<at::Float8_e4m3fn>(avg_M);
|
||||
const bool use_brgemm = can_use_brgemm<packed_t>(avg_M);
|
||||
|
||||
int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N);
|
||||
|
||||
@@ -57,14 +71,15 @@ void fused_experts_fp8_kernel_impl(
|
||||
int tid = get_thread_num();
|
||||
scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K;
|
||||
|
||||
loop_2d<at::Float8_e4m3fn>(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
loop_2d<packed_t>(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N);
|
||||
|
||||
// B shape [K, n_size] in vnni format
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e + nb * BLOCK_N * stride_n;
|
||||
const float* __restrict__ Bs =
|
||||
w1s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K;
|
||||
const packed_t* __restrict__ B = packed_w1 + expert_id * stride_e + nb * BLOCK_N * stride_n;
|
||||
const param_t* __restrict__ Bs =
|
||||
w1s + expert_id * scale_size_N * scale_size_K + scale_offset_per_block(nb) * scale_size_K;
|
||||
const float* __restrict__ B_bias = with_bias ? w1_bias + expert_id * 2 * N + nb * BLOCK_N : nullptr;
|
||||
|
||||
// do unpacking for the first row or a new expert
|
||||
int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1];
|
||||
@@ -88,6 +103,7 @@ void fused_experts_fp8_kernel_impl(
|
||||
/* C */ ic0 + offset * 2 * N + nb * BLOCK_N,
|
||||
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K,
|
||||
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N,
|
||||
/* Bbias */ B_bias,
|
||||
/* scale */ Bs,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
@@ -106,12 +122,20 @@ void fused_experts_fp8_kernel_impl(
|
||||
});
|
||||
|
||||
// stage 1.5: intermediate_cache1 = silu(intermediate_cache0)
|
||||
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N);
|
||||
}
|
||||
});
|
||||
|
||||
if (act_func == CPUAcTMethod::silu_and_mul) {
|
||||
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N);
|
||||
}
|
||||
});
|
||||
} else if (act_func == CPUAcTMethod::swiglu) {
|
||||
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
clamp_sigmoid_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, N, alpha, limit);
|
||||
clamp_sigmoid_and_mul_stub(ic1 + m * N + N / 2, ic0 + m * 2 * N + N, N, alpha, limit);
|
||||
}
|
||||
});
|
||||
}
|
||||
// stage 2: intermediate_cache2 = intermediate_cache1 @ w2
|
||||
// w2 : [E, K, N] as [E, OC, IC]
|
||||
const int64_t OC = K; // rename K as OC
|
||||
@@ -120,15 +144,16 @@ void fused_experts_fp8_kernel_impl(
|
||||
const int64_t NB2 = div_up(OC, BLOCK_N);
|
||||
scale_size_N = div_up(K, block_size_N);
|
||||
scale_size_K = div_up(N, block_size_K);
|
||||
const int64_t stride_e2 = OC * IC;
|
||||
const int64_t stride_oc = IC;
|
||||
const int64_t packed_IC = get_row_size<packed_t>(IC);
|
||||
const int64_t stride_e2 = OC * packed_IC;
|
||||
const int64_t stride_oc = packed_IC;
|
||||
|
||||
// parallel on [MB2, NB2]
|
||||
parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
int tid = get_thread_num();
|
||||
alignas(64) scalar_t C[BLOCK_M * BLOCK_K];
|
||||
|
||||
loop_2d<at::Float8_e4m3fn>(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
loop_2d<packed_t>(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N);
|
||||
|
||||
@@ -139,9 +164,10 @@ void fused_experts_fp8_kernel_impl(
|
||||
|
||||
// B shape [IC, n_size] in vnni format
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const at::Float8_e4m3fn* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc;
|
||||
const float* __restrict__ Bs =
|
||||
w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K;
|
||||
const packed_t* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc;
|
||||
const param_t* __restrict__ Bs =
|
||||
w2s + expert_id * scale_size_N * scale_size_K + scale_offset_per_block(nb) * scale_size_K;
|
||||
const float* __restrict__ B_bias = with_bias ? w2_bias + expert_id * OC + nb * BLOCK_N : nullptr;
|
||||
|
||||
// do unpacking for the first row or a new expert
|
||||
int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1];
|
||||
@@ -153,6 +179,7 @@ void fused_experts_fp8_kernel_impl(
|
||||
/* C */ C,
|
||||
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC,
|
||||
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N,
|
||||
/* Bbias */ B_bias,
|
||||
/* scale */ Bs,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
@@ -177,7 +204,6 @@ void fused_experts_fp8_kernel_impl(
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
|
||||
// stage 3: out = intermediate_cache2.sum(dim=1)
|
||||
// from [M, topk, K] to [M, K]
|
||||
at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) {
|
||||
@@ -187,35 +213,43 @@ void fused_experts_fp8_kernel_impl(
|
||||
});
|
||||
}
|
||||
|
||||
#define INSTANTIATE_MOE_FP8_TEMPLATE(TYPE) \
|
||||
template void fused_experts_fp8_kernel_impl<TYPE>( \
|
||||
TYPE* __restrict__ output, \
|
||||
TYPE* __restrict__ ic0, \
|
||||
TYPE* __restrict__ ic1, \
|
||||
TYPE* __restrict__ ic2, \
|
||||
TYPE* __restrict__ A_tmp, \
|
||||
TYPE* __restrict__ B_tmp, \
|
||||
float* __restrict__ C_tmp, \
|
||||
const TYPE* __restrict__ input, \
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w1, \
|
||||
const at::Float8_e4m3fn* __restrict__ packed_w2, \
|
||||
const float* __restrict__ w1s, \
|
||||
const float* __restrict__ w2s, \
|
||||
int64_t block_size_N, \
|
||||
int64_t block_size_K, \
|
||||
const float* __restrict__ topk_weights, \
|
||||
const int32_t* __restrict__ sorted_ids, \
|
||||
const int32_t* __restrict__ expert_ids, \
|
||||
const int32_t* __restrict__ offsets, \
|
||||
int64_t M, \
|
||||
int64_t N, \
|
||||
int64_t K, \
|
||||
int64_t E, \
|
||||
int64_t topk, \
|
||||
int64_t num_tokens_post_pad)
|
||||
#define INSTANTIATE_MOE_FP_TEMPLATE(TYPE1, TYPE2, TYPE3, IS_MXFP4) \
|
||||
template void fused_experts_fp_kernel_impl<TYPE1, TYPE2, TYPE3, IS_MXFP4>( \
|
||||
TYPE1* __restrict__ output, \
|
||||
TYPE1* __restrict__ ic0, \
|
||||
TYPE1* __restrict__ ic1, \
|
||||
TYPE1* __restrict__ ic2, \
|
||||
TYPE1* __restrict__ A_tmp, \
|
||||
TYPE1* __restrict__ B_tmp, \
|
||||
float* __restrict__ C_tmp, \
|
||||
const TYPE1* __restrict__ input, \
|
||||
const TYPE2* __restrict__ packed_w1, \
|
||||
const TYPE2* __restrict__ packed_w2, \
|
||||
const float* __restrict__ w1_bias, \
|
||||
const float* __restrict__ w2_bias, \
|
||||
const TYPE3* __restrict__ w1s, \
|
||||
const TYPE3* __restrict__ w2s, \
|
||||
int64_t block_size_N, \
|
||||
int64_t block_size_K, \
|
||||
const float* __restrict__ topk_weights, \
|
||||
const int32_t* __restrict__ sorted_ids, \
|
||||
const int32_t* __restrict__ expert_ids, \
|
||||
const int32_t* __restrict__ offsets, \
|
||||
int64_t M, \
|
||||
int64_t N, \
|
||||
int64_t K, \
|
||||
int64_t E, \
|
||||
int64_t topk, \
|
||||
int64_t num_tokens_post_pad, \
|
||||
float alpha, \
|
||||
float limit, \
|
||||
CPUAcTMethod act_func, \
|
||||
bool with_bias)
|
||||
|
||||
INSTANTIATE_MOE_FP8_TEMPLATE(at::BFloat16);
|
||||
INSTANTIATE_MOE_FP8_TEMPLATE(at::Half);
|
||||
INSTANTIATE_MOE_FP_TEMPLATE(at::BFloat16, at::Float8_e4m3fn, float, false);
|
||||
INSTANTIATE_MOE_FP_TEMPLATE(at::Half, at::Float8_e4m3fn, float, false);
|
||||
INSTANTIATE_MOE_FP_TEMPLATE(at::BFloat16, uint8_t, uint8_t, true);
|
||||
INSTANTIATE_MOE_FP_TEMPLATE(at::Half, uint8_t, uint8_t, true);
|
||||
|
||||
template <typename scalar_t>
|
||||
void shared_expert_fp8_kernel_impl(
|
||||
@@ -266,6 +300,7 @@ void shared_expert_fp8_kernel_impl(
|
||||
/* C */ ic0 + mb * BLOCK_M * 2 * N + nb * BLOCK_N,
|
||||
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K,
|
||||
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N,
|
||||
/* Bbias */ nullptr,
|
||||
/* scale */ w1s + (nb / blocks_n_per_group) * scale_size_K,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
@@ -317,6 +352,7 @@ void shared_expert_fp8_kernel_impl(
|
||||
/* C */ C,
|
||||
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC,
|
||||
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N,
|
||||
/* Bbias */ nullptr,
|
||||
/* scale */ w2s + (nb / blocks_n_per_group) * scale_size_K,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
|
||||
#include <ATen/cpu/vec/functional.h>
|
||||
#include <ATen/cpu/vec/vec.h>
|
||||
|
||||
#include <immintrin.h>
|
||||
namespace {
|
||||
|
||||
using namespace at::vec;
|
||||
@@ -21,6 +21,15 @@ inline Vectorized<scalar_t> convert_from_float_ext(const Vectorized<float>& a, c
|
||||
return at::vec::convert_from_float<scalar_t>(a, b);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void convert_from_float_and_store(scalar_t* out, const Vectorized<float>& a) {
|
||||
float out_buffer[at::vec::Vectorized<float>::size()];
|
||||
a.store(out_buffer);
|
||||
for (int i = 0; i < 16; i++) {
|
||||
out[i] = (scalar_t)out_buffer[i];
|
||||
}
|
||||
}
|
||||
|
||||
// allow f16, bf16
|
||||
template <typename scalar_t, typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 1>
|
||||
inline std::tuple<Vectorized<float>, Vectorized<float>> load_float_vec2(const scalar_t* __restrict__ data) {
|
||||
@@ -50,6 +59,11 @@ convert_from_float_ext<at::BFloat16>(const Vectorized<float>& a, const Vectorize
|
||||
return (__m512i)(_mm512_cvtne2ps_pbh(__m512(b), __m512(a)));
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void convert_from_float_and_store<at::BFloat16>(at::BFloat16* out, const Vectorized<float>& a) {
|
||||
_mm256_storeu_si256((__m256i*)out, (__m256i)(_mm512_cvtneps_pbh(__m512(a))));
|
||||
}
|
||||
|
||||
#define CVT_BF16_TO_FP32(a) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16))
|
||||
|
||||
#define CVT_FP16_TO_FP32(a) _mm512_cvtph_ps(a)
|
||||
@@ -125,7 +139,6 @@ inline __m512bh CVT_FP8_TO_BF16(__m256i a) {
|
||||
return cvt_e4m3_bf16_intrinsic_with_denorm(a);
|
||||
#endif
|
||||
}
|
||||
|
||||
// faster version of float8_e4m3fn conversion to bfloat16
|
||||
//
|
||||
// we mapped cuda implementation from below link and vectorized with avx512:
|
||||
|
||||
@@ -64,6 +64,8 @@ at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2,
|
||||
|
||||
at::Tensor convert_weight_packed(at::Tensor& weight);
|
||||
|
||||
at::Tensor convert_scale_packed(at::Tensor& scale);
|
||||
|
||||
at::Tensor fused_experts_cpu(
|
||||
at::Tensor& hidden_states, at::Tensor& w1, at::Tensor& w2,
|
||||
at::Tensor& topk_weights, at::Tensor& topk_ids, bool inplace,
|
||||
@@ -71,7 +73,11 @@ at::Tensor fused_experts_cpu(
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<at::Tensor>& w1_zero,
|
||||
const std::optional<at::Tensor>& w2_zero,
|
||||
const std::optional<std::vector<int64_t>> block_size, bool is_vnni);
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
const std::optional<at::Tensor>& w1_bias,
|
||||
const std::optional<at::Tensor>& w2_bias,
|
||||
const std::optional<double>& alpha, const std::optional<double>& limit,
|
||||
bool is_vnni);
|
||||
|
||||
at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2,
|
||||
at::Tensor& scales2,
|
||||
@@ -404,11 +410,15 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
ops.impl("weight_packed_linear", torch::kCPU, &weight_packed_linear);
|
||||
ops.def("convert_weight_packed(Tensor! weight) -> Tensor");
|
||||
ops.impl("convert_weight_packed", torch::kCPU, &convert_weight_packed);
|
||||
ops.def("convert_scale_packed(Tensor! scale) -> Tensor");
|
||||
ops.impl("convert_scale_packed", torch::kCPU, &convert_scale_packed);
|
||||
ops.def(
|
||||
"fused_experts_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor "
|
||||
"topk_weights, Tensor topk_ids, bool "
|
||||
"inplace, int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
|
||||
"Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, bool is_vnni) -> "
|
||||
"Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, "
|
||||
"Tensor? w1_bias, Tensor? w2_bias, float? alpha, float? limit, "
|
||||
"bool is_vnni) -> "
|
||||
"Tensor");
|
||||
ops.impl("fused_experts_cpu", torch::kCPU, &fused_experts_cpu);
|
||||
ops.def(
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for CPU FP8 W8A16 block-scaled fused MoE kernel."""
|
||||
|
||||
import math
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
import vllm._custom_ops as ops # noqa: E402
|
||||
|
||||
if not hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
pytest.skip("fused_experts_cpu op not available", allow_module_level=True)
|
||||
|
||||
|
||||
BLOCK_SIZE = [128, 128] # [block_n, block_k]
|
||||
|
||||
# FP8 weight generation parameters
|
||||
_FP8_INFO = torch.finfo(torch.float8_e4m3fn)
|
||||
FP8_SCALE = _FP8_INFO.max # 448.0
|
||||
FACTOR_FOR_SCALE = 1e-3
|
||||
|
||||
# Tolerance for FP8 W8A16
|
||||
FP8_W8A16_ATOL = 1e-2
|
||||
FP8_W8A16_RTOL = 1e-2
|
||||
|
||||
|
||||
def _silu_and_mul(x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
return F.silu(x[..., :d]) * x[..., d:]
|
||||
|
||||
|
||||
def _block_dequant_weight(
|
||||
weight: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""Block-dequantize FP8 weight [E, N, K] → float [E, N, K].
|
||||
|
||||
Each (block_n × block_k) tile is multiplied by its per-block scale.
|
||||
"""
|
||||
E, N, K = weight.shape
|
||||
block_n, block_k = block_size
|
||||
pad_N = (block_n - N % block_n) % block_n
|
||||
pad_K = (block_k - K % block_k) % block_k
|
||||
|
||||
if pad_N > 0 or pad_K > 0:
|
||||
weight = F.pad(weight, (0, pad_K, 0, pad_N))
|
||||
|
||||
n_tiles = math.ceil(N / block_n)
|
||||
k_tiles = math.ceil(K / block_k)
|
||||
|
||||
weight_block = (
|
||||
weight.view(E, n_tiles, block_n, k_tiles, block_k)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.float()
|
||||
.contiguous()
|
||||
)
|
||||
weight_scaled = (
|
||||
(weight_block * scales.view(E, n_tiles, k_tiles, 1, 1))
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
)
|
||||
if pad_N > 0 or pad_K > 0:
|
||||
weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K)
|
||||
weight_scaled = weight_scaled[..., :N, :K].contiguous()
|
||||
else:
|
||||
weight_scaled = weight_scaled.view(E, N, K)
|
||||
return weight_scaled
|
||||
|
||||
|
||||
def ref_w8a16_block_fp8_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_s: torch.Tensor,
|
||||
w2_s: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""Reference FP8 W8A16 block-scaled fused MoE in pure torch.
|
||||
|
||||
Steps:
|
||||
1. Block-dequant FP8 weights → float
|
||||
2. For each expert: matmul → SiLU+Mul → matmul
|
||||
3. Weighted sum across top-k experts
|
||||
"""
|
||||
B, D = a.shape
|
||||
topk = topk_ids.size(1)
|
||||
|
||||
w1_dq = _block_dequant_weight(w1, w1_s, block_size)
|
||||
w2_dq = _block_dequant_weight(w2, w2_s, block_size)
|
||||
|
||||
a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float()
|
||||
out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32)
|
||||
|
||||
topk_weight_flat = topk_weight.view(-1)
|
||||
topk_ids_flat = topk_ids.view(-1)
|
||||
|
||||
for i in range(w1_dq.shape[0]):
|
||||
mask = topk_ids_flat == i
|
||||
if mask.sum():
|
||||
ic0 = torch.matmul(a_exp[mask], w1_dq[i].transpose(0, 1))
|
||||
ic1 = _silu_and_mul(ic0)
|
||||
out[mask] = torch.matmul(ic1, w2_dq[i].transpose(0, 1))
|
||||
|
||||
return (
|
||||
(out.view(B, -1, w2_dq.shape[1]) * topk_weight_flat.view(B, -1, 1))
|
||||
.sum(dim=1)
|
||||
.to(a.dtype)
|
||||
)
|
||||
|
||||
|
||||
def _make_fp8_moe_weights(
|
||||
E: int,
|
||||
N: int,
|
||||
K: int,
|
||||
block_size: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Generate random FP8 MoE weights with random block scales.
|
||||
|
||||
Weight generation follows SGLang: ``randn * FP8_SCALE`` → clamp → cast.
|
||||
Scales are small random values (``FACTOR_FOR_SCALE``), independent of
|
||||
the actual weight magnitudes — this is sufficient to test the kernel's
|
||||
block-dequant + matmul correctness.
|
||||
|
||||
Returns: (w1, w2, w1_s, w2_s)
|
||||
"""
|
||||
block_n, block_k = block_size
|
||||
|
||||
w1 = (
|
||||
(torch.randn(E, 2 * N, K) * FP8_SCALE)
|
||||
.clamp(min=-FP8_SCALE, max=FP8_SCALE)
|
||||
.to(torch.float8_e4m3fn)
|
||||
)
|
||||
w2 = (
|
||||
(torch.randn(E, K, N) * FP8_SCALE)
|
||||
.clamp(min=-FP8_SCALE, max=FP8_SCALE)
|
||||
.to(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
w1_s = (
|
||||
torch.randn(E, math.ceil(2 * N / block_n), math.ceil(K / block_k))
|
||||
* FACTOR_FOR_SCALE
|
||||
)
|
||||
w2_s = (
|
||||
torch.randn(E, math.ceil(K / block_n), math.ceil(N / block_k))
|
||||
* FACTOR_FOR_SCALE
|
||||
)
|
||||
return w1, w2, w1_s, w2_s
|
||||
|
||||
|
||||
def _prepack_experts(w: torch.Tensor) -> torch.Tensor:
|
||||
"""VNNI-prepack each expert's weight via ``convert_weight_packed``."""
|
||||
return torch.stack(
|
||||
[torch.ops._C.convert_weight_packed(w[e]) for e in range(w.shape[0])]
|
||||
)
|
||||
|
||||
|
||||
NUM_TOKENS = [1, 2, 64, 121]
|
||||
# (M, intermediate_size N, hidden_size K, num_experts E, topk)
|
||||
MoE_CONFIGS = [
|
||||
(256, 512, 8, 2),
|
||||
(256, 512, 8, 4),
|
||||
(512, 256, 8, 2),
|
||||
(512, 256, 8, 4),
|
||||
(512, 512, 8, 2),
|
||||
(512, 512, 8, 4),
|
||||
(768, 2048, 8, 2),
|
||||
(768, 2048, 8, 4),
|
||||
(768, 2048, 128, 8),
|
||||
]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("N,K,E,topk", MoE_CONFIGS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
"""Test fused_experts_cpu FP8 W8A16 against dequantised torch reference."""
|
||||
set_random_seed(seed)
|
||||
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K)
|
||||
w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE)
|
||||
|
||||
score = torch.randn(M, E, dtype=torch.bfloat16)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
ref_out = ref_w8a16_block_fp8_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
w1_s,
|
||||
w2_s,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
BLOCK_SIZE,
|
||||
)
|
||||
|
||||
pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)
|
||||
|
||||
# Test inplace=False against reference
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False,
|
||||
ops.CPUQuantMethod.FP8_W8A16,
|
||||
w1_s,
|
||||
w2_s,
|
||||
None,
|
||||
None,
|
||||
BLOCK_SIZE,
|
||||
True,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ref_out.bfloat16(),
|
||||
out,
|
||||
atol=FP8_W8A16_ATOL,
|
||||
rtol=FP8_W8A16_RTOL,
|
||||
)
|
||||
|
||||
# Test inplace=True produces identical output
|
||||
out_inplace = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
True,
|
||||
ops.CPUQuantMethod.FP8_W8A16,
|
||||
w1_s,
|
||||
w2_s,
|
||||
None,
|
||||
None,
|
||||
BLOCK_SIZE,
|
||||
True,
|
||||
)
|
||||
torch.testing.assert_close(out_inplace, out, atol=0, rtol=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
@@ -0,0 +1,500 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for CPU quantized fused MoE kernels (FP8 W8A16 and MXFP4 W4A16)."""
|
||||
|
||||
import math
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
import vllm._custom_ops as ops # noqa: E402
|
||||
|
||||
if not hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
pytest.skip("fused_experts_cpu op not available", allow_module_level=True)
|
||||
|
||||
|
||||
def _silu_and_mul(x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
return F.silu(x[..., :d]) * x[..., d:]
|
||||
|
||||
|
||||
def _prepack_experts(w: torch.Tensor) -> torch.Tensor:
|
||||
"""VNNI-prepack expert weights via ``convert_weight_packed``."""
|
||||
return torch.ops._C.convert_weight_packed(w)
|
||||
|
||||
|
||||
# FP8 W8A16 block-scaled fused MoE
|
||||
|
||||
BLOCK_SIZE = [128, 128] # [block_n, block_k]
|
||||
|
||||
_FP8_INFO = torch.finfo(torch.float8_e4m3fn)
|
||||
FP8_SCALE = _FP8_INFO.max # 448.0
|
||||
FACTOR_FOR_SCALE = 1e-3
|
||||
|
||||
|
||||
def _block_dequant_weight(
|
||||
weight: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""Block-dequantize FP8 weight [E, N, K] -> float [E, N, K]."""
|
||||
E, N, K = weight.shape
|
||||
block_n, block_k = block_size
|
||||
pad_N = (block_n - N % block_n) % block_n
|
||||
pad_K = (block_k - K % block_k) % block_k
|
||||
|
||||
if pad_N > 0 or pad_K > 0:
|
||||
weight = F.pad(weight, (0, pad_K, 0, pad_N))
|
||||
|
||||
n_tiles = math.ceil(N / block_n)
|
||||
k_tiles = math.ceil(K / block_k)
|
||||
|
||||
weight_block = (
|
||||
weight.view(E, n_tiles, block_n, k_tiles, block_k)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.float()
|
||||
.contiguous()
|
||||
)
|
||||
weight_scaled = (
|
||||
(weight_block * scales.view(E, n_tiles, k_tiles, 1, 1))
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
)
|
||||
if pad_N > 0 or pad_K > 0:
|
||||
weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K)
|
||||
weight_scaled = weight_scaled[..., :N, :K].contiguous()
|
||||
else:
|
||||
weight_scaled = weight_scaled.view(E, N, K)
|
||||
return weight_scaled
|
||||
|
||||
|
||||
def ref_w8a16_block_fp8_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_s: torch.Tensor,
|
||||
w2_s: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""Reference FP8 W8A16 block-scaled fused MoE in pure torch."""
|
||||
B, D = a.shape
|
||||
topk = topk_ids.size(1)
|
||||
|
||||
w1_dq = _block_dequant_weight(w1, w1_s, block_size)
|
||||
w2_dq = _block_dequant_weight(w2, w2_s, block_size)
|
||||
|
||||
a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float()
|
||||
out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32)
|
||||
|
||||
topk_weight_flat = topk_weight.view(-1)
|
||||
topk_ids_flat = topk_ids.view(-1)
|
||||
|
||||
for i in range(w1_dq.shape[0]):
|
||||
mask = topk_ids_flat == i
|
||||
if mask.sum():
|
||||
ic0 = torch.matmul(a_exp[mask], w1_dq[i].transpose(0, 1))
|
||||
ic1 = _silu_and_mul(ic0)
|
||||
out[mask] = torch.matmul(ic1, w2_dq[i].transpose(0, 1))
|
||||
|
||||
return (
|
||||
(out.view(B, -1, w2_dq.shape[1]) * topk_weight_flat.view(B, -1, 1))
|
||||
.sum(dim=1)
|
||||
.to(a.dtype)
|
||||
)
|
||||
|
||||
|
||||
def _make_fp8_moe_weights(
|
||||
E: int,
|
||||
N: int,
|
||||
K: int,
|
||||
block_size: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Generate random FP8 MoE weights with random block scales."""
|
||||
block_n, block_k = block_size
|
||||
|
||||
w1 = (
|
||||
(torch.randn(E, 2 * N, K) * FP8_SCALE)
|
||||
.clamp(min=-FP8_SCALE, max=FP8_SCALE)
|
||||
.to(torch.float8_e4m3fn)
|
||||
)
|
||||
w2 = (
|
||||
(torch.randn(E, K, N) * FP8_SCALE)
|
||||
.clamp(min=-FP8_SCALE, max=FP8_SCALE)
|
||||
.to(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
w1_s = (
|
||||
torch.randn(E, math.ceil(2 * N / block_n), math.ceil(K / block_k))
|
||||
* FACTOR_FOR_SCALE
|
||||
)
|
||||
w2_s = (
|
||||
torch.randn(E, math.ceil(K / block_n), math.ceil(N / block_k))
|
||||
* FACTOR_FOR_SCALE
|
||||
)
|
||||
return w1, w2, w1_s, w2_s
|
||||
|
||||
|
||||
FP8_NUM_TOKENS = [1, 2, 64, 121]
|
||||
FP8_MOE_CONFIGS = [
|
||||
(256, 512, 8, 2),
|
||||
(256, 512, 8, 4),
|
||||
(512, 256, 8, 2),
|
||||
(512, 256, 8, 4),
|
||||
(512, 512, 8, 2),
|
||||
(512, 512, 8, 4),
|
||||
(768, 2048, 8, 2),
|
||||
(768, 2048, 8, 4),
|
||||
(768, 2048, 128, 8),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", FP8_NUM_TOKENS)
|
||||
@pytest.mark.parametrize("N,K,E,topk", FP8_MOE_CONFIGS)
|
||||
@pytest.mark.parametrize("seed", [0])
|
||||
def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
"""Test fused_experts_cpu FP8 W8A16 against dequantised torch reference."""
|
||||
set_random_seed(seed)
|
||||
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K)
|
||||
w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE)
|
||||
|
||||
score = torch.randn(M, E, dtype=torch.bfloat16)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
ref_out = ref_w8a16_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, BLOCK_SIZE
|
||||
)
|
||||
|
||||
pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)
|
||||
|
||||
# Test inplace=False against reference
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False,
|
||||
ops.CPUQuantMethod.FP8_W8A16,
|
||||
w1_s,
|
||||
w2_s,
|
||||
None,
|
||||
None,
|
||||
BLOCK_SIZE,
|
||||
is_vnni=True,
|
||||
)
|
||||
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
# Test inplace=True produces identical output
|
||||
out_inplace = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
True,
|
||||
ops.CPUQuantMethod.FP8_W8A16,
|
||||
w1_s,
|
||||
w2_s,
|
||||
None,
|
||||
None,
|
||||
BLOCK_SIZE,
|
||||
is_vnni=True,
|
||||
)
|
||||
torch.testing.assert_close(out_inplace, out, atol=0, rtol=0)
|
||||
|
||||
|
||||
# MXFP4 W4A16 fused MoE
|
||||
|
||||
|
||||
class MXFP4QuantizeUtil:
|
||||
"""MXFP4 quantization utility."""
|
||||
|
||||
E2M1_max = 6.0
|
||||
E2M1_values = [0, 0.5, 1, 1.5, 2, 3, 4, 6]
|
||||
E2M1_bounds = torch.tensor([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5])
|
||||
block_size = 32
|
||||
|
||||
@classmethod
|
||||
def quantize(cls, input: torch.Tensor) -> tuple:
|
||||
"""Quantize BF16 tensor to MXFP4 packed uint8 format."""
|
||||
|
||||
def cast_fp4(x):
|
||||
sign = torch.sign(x)
|
||||
sign_bit = (2 - sign) // 2
|
||||
ord_ = torch.sum(
|
||||
(x.abs().unsqueeze(-1) - cls.E2M1_bounds.to(x.device)) > 0, dim=-1
|
||||
)
|
||||
fp4_val = (sign_bit * 0b1000 + ord_).to(torch.uint8)
|
||||
return fp4_val
|
||||
|
||||
def fuse_uint4_to_uint8(x):
|
||||
left_side = x[..., 0::2]
|
||||
right_side = x[..., 1::2]
|
||||
new_data = right_side.clone() << 4
|
||||
new_data[..., : left_side.shape[-1]] += left_side
|
||||
return new_data
|
||||
|
||||
original_shape = input.shape
|
||||
input = input.view(-1, cls.block_size)
|
||||
input_amax = input.abs().max(dim=-1, keepdim=True).values
|
||||
descale = input_amax / cls.E2M1_max
|
||||
min_value = torch.tensor(-127.0, device=descale.device)
|
||||
e8m0_scale = torch.ceil(torch.maximum(torch.log2(descale), min_value))
|
||||
|
||||
input = (input / torch.exp2(e8m0_scale)).view(original_shape)
|
||||
input_q = cast_fp4(input)
|
||||
input_q = fuse_uint4_to_uint8(input_q)
|
||||
e8m0_scale = (e8m0_scale + 127).to(torch.uint8)
|
||||
return input_q, e8m0_scale
|
||||
|
||||
@classmethod
|
||||
def dequantize(cls, quantized_data, dtype: torch.dtype, scale):
|
||||
"""Dequantize MXFP4 packed tensor back to float."""
|
||||
|
||||
def unfuse_uint8_to_uint4(x):
|
||||
left_side = x & 0x0F
|
||||
right_side = (x >> 4) & 0x0F
|
||||
shape = list(x.shape)
|
||||
shape[-1] = shape[-1] * 2
|
||||
result = torch.zeros(shape, dtype=torch.uint8, device=x.device)
|
||||
result[..., 0::2] = left_side
|
||||
result[..., 1::2] = right_side
|
||||
return result
|
||||
|
||||
e8m0_scale = scale
|
||||
x_unfused = unfuse_uint8_to_uint4(quantized_data)
|
||||
sign = 1 - 2 * ((x_unfused & 0b1000) >> 3).to(torch.float32)
|
||||
magnitude = (x_unfused & 0b0111).to(torch.long)
|
||||
values = torch.tensor(cls.E2M1_values, device=quantized_data.device)
|
||||
original_shape = magnitude.shape
|
||||
x_float = values[magnitude.reshape(-1)].reshape(original_shape)
|
||||
x_float = sign.float() * x_float
|
||||
x_float = x_float.reshape(-1, cls.block_size)
|
||||
scale_factor = torch.exp2(e8m0_scale.float() - 127)
|
||||
scale_factor = scale_factor.reshape(-1, 1)
|
||||
x_float = x_float * scale_factor
|
||||
return x_float.reshape(original_shape).to(dtype)
|
||||
|
||||
|
||||
def _swiglu(x: torch.Tensor, alpha: float, limit: float) -> torch.Tensor:
|
||||
"""SwigLU activation used in GPT-OSS.
|
||||
Input is interleaved: [gate_0, up_0, gate_1, up_1, ...] in last dim.
|
||||
"""
|
||||
gate = x[..., 0::2]
|
||||
up = x[..., 1::2]
|
||||
gate_clamped = torch.clamp(gate, max=limit)
|
||||
up_clamped = torch.clamp(up, min=-limit, max=limit)
|
||||
return gate_clamped * torch.sigmoid(alpha * gate_clamped) * (up_clamped + 1)
|
||||
|
||||
|
||||
def ref_mxfp4_fused_moe(
|
||||
a: torch.Tensor,
|
||||
w1_dq: torch.Tensor,
|
||||
w2_dq: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk: int,
|
||||
) -> torch.Tensor:
|
||||
"""Reference MXFP4 fused MoE with SiLU activation."""
|
||||
B, D = a.shape
|
||||
a_f = a.float()
|
||||
out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32)
|
||||
topk_ids_flat = topk_ids.view(-1)
|
||||
|
||||
for i in range(w1_dq.shape[0]):
|
||||
mask = topk_ids_flat == i
|
||||
if mask.sum() == 0:
|
||||
continue
|
||||
token_indices = torch.where(mask)[0]
|
||||
source_indices = token_indices // topk
|
||||
ic0 = torch.matmul(a_f[source_indices], w1_dq[i].float().T)
|
||||
ic1 = _silu_and_mul(ic0)
|
||||
out[mask] = torch.matmul(ic1, w2_dq[i].float().T)
|
||||
|
||||
return (out.view(B, topk, -1) * topk_weight.unsqueeze(-1)).sum(dim=1).to(a.dtype)
|
||||
|
||||
|
||||
def ref_mxfp4_fused_moe_gptoss(
|
||||
a: torch.Tensor,
|
||||
w1_dq: torch.Tensor,
|
||||
w2_dq: torch.Tensor,
|
||||
w1_bias: torch.Tensor,
|
||||
w2_bias: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
alpha: float,
|
||||
limit: float,
|
||||
) -> torch.Tensor:
|
||||
"""Reference MXFP4 fused MoE with SwigLU+bias (GPT-OSS style)."""
|
||||
B, D = a.shape
|
||||
topk = topk_ids.shape[1]
|
||||
a_f = a.float()
|
||||
E = w1_dq.shape[0]
|
||||
out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32)
|
||||
topk_ids_flat = topk_ids.view(-1)
|
||||
|
||||
for i in range(E):
|
||||
mask = topk_ids_flat == i
|
||||
if mask.sum() == 0:
|
||||
continue
|
||||
token_indices = torch.where(mask)[0]
|
||||
source_indices = token_indices // topk
|
||||
ic0 = torch.matmul(a_f[source_indices], w1_dq[i].float().T)
|
||||
ic0 = ic0 + w1_bias[i].float()
|
||||
ic1 = _swiglu(ic0, alpha, limit)
|
||||
ic2 = torch.matmul(ic1, w2_dq[i].float().T)
|
||||
ic2 = ic2 + w2_bias[i].float()
|
||||
out[mask] = ic2
|
||||
|
||||
return (out.view(B, topk, -1) * topk_weight.unsqueeze(-1)).sum(dim=1).to(a.dtype)
|
||||
|
||||
|
||||
def _prepack_mxfp4_experts(
|
||||
w: torch.Tensor, w_scale: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""VNNI-prepack MXFP4 weights and repack scales."""
|
||||
packed_w = torch.ops._C.convert_weight_packed(w)
|
||||
packed_s = torch.ops._C.convert_scale_packed(w_scale)
|
||||
return packed_w, packed_s
|
||||
|
||||
|
||||
MXFP4_NUM_TOKENS = [1, 2, 32, 121]
|
||||
MXFP4_MOE_CONFIGS = [
|
||||
(128, 128, 4, 2),
|
||||
(256, 256, 8, 4),
|
||||
(352, 256, 8, 4),
|
||||
(512, 320, 8, 4),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", MXFP4_NUM_TOKENS)
|
||||
@pytest.mark.parametrize("N,K,E,topk", MXFP4_MOE_CONFIGS)
|
||||
@pytest.mark.parametrize("seed", [0])
|
||||
def test_mxfp4_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
"""Test fused_experts_mxfp4_cpu against dequantized torch reference."""
|
||||
set_random_seed(seed)
|
||||
dtype = torch.bfloat16
|
||||
|
||||
a = torch.randn(M, K, dtype=dtype) / 10
|
||||
|
||||
# Generate and quantize weights
|
||||
w1_bf16 = torch.randn(E, 2 * N, K, dtype=dtype) / 10
|
||||
w1q, w1s = MXFP4QuantizeUtil.quantize(w1_bf16)
|
||||
w1s = w1s.reshape(E, 2 * N, K // 32)
|
||||
w1dq = MXFP4QuantizeUtil.dequantize(w1q, dtype, w1s)
|
||||
|
||||
w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10
|
||||
w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16)
|
||||
w2s = w2s.reshape(E, K, N // 32)
|
||||
w2dq = MXFP4QuantizeUtil.dequantize(w2q, dtype, w2s)
|
||||
|
||||
# Routing
|
||||
score = torch.randn(M, E, dtype=dtype)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
# Reference
|
||||
ref_out = ref_mxfp4_fused_moe(a, w1dq, w2dq, topk_weight, topk_ids, topk)
|
||||
|
||||
# Pack weights for kernel
|
||||
pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
|
||||
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
|
||||
|
||||
# Kernel
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
ops.CPUQuantMethod.MXFP4,
|
||||
pw1s, # w1_scale
|
||||
pw2s, # w2_scale
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
None, # block_size
|
||||
)
|
||||
|
||||
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 32])
|
||||
@pytest.mark.parametrize("N,K,E,topk", [(128, 128, 4, 2), (64, 64, 4, 2)])
|
||||
@pytest.mark.parametrize("seed", [0])
|
||||
def test_mxfp4_cpu_fused_moe_bias_swiglu(M, N, K, E, topk, seed):
|
||||
"""Test fused_experts_mxfp4_cpu with bias and SwigLU activation (GPT-OSS)."""
|
||||
set_random_seed(seed)
|
||||
dtype = torch.bfloat16
|
||||
alpha = 1.702
|
||||
limit = 7.0
|
||||
|
||||
a = torch.randn(M, K, dtype=dtype) / 10
|
||||
|
||||
# Generate and quantize weights
|
||||
w1_bf16 = torch.randn(E, 2 * N, K, dtype=dtype) / 10
|
||||
w1q, w1s = MXFP4QuantizeUtil.quantize(w1_bf16)
|
||||
w1s = w1s.reshape(E, 2 * N, K // 32)
|
||||
w1dq = MXFP4QuantizeUtil.dequantize(w1q, dtype, w1s)
|
||||
w1_b = torch.randn(E, 2 * N, dtype=torch.float32) / 10
|
||||
|
||||
w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10
|
||||
w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16)
|
||||
w2s = w2s.reshape(E, K, N // 32)
|
||||
w2dq = MXFP4QuantizeUtil.dequantize(w2q, dtype, w2s)
|
||||
w2_b = torch.randn(E, K, dtype=torch.float32) / 10
|
||||
|
||||
# Routing
|
||||
score = torch.randn(M, E, dtype=dtype)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
# Reference
|
||||
ref_out = ref_mxfp4_fused_moe_gptoss(
|
||||
a, w1dq, w2dq, w1_b, w2_b, topk_weight, topk_ids, alpha, limit
|
||||
)
|
||||
|
||||
# Pack weights for kernel
|
||||
pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
|
||||
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
|
||||
|
||||
# Kernel
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
ops.CPUQuantMethod.MXFP4,
|
||||
pw1s, # w1_scale
|
||||
pw2s, # w2_scale
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
None, # block_size
|
||||
w1_bias=w1_b,
|
||||
w2_bias=w2_b,
|
||||
alpha=alpha,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
@@ -15,6 +15,7 @@ MODELS = [
|
||||
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
|
||||
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear
|
||||
"Qwen/Qwen3-30B-A3B-FP8", # FP8 W8A16 block-quantized MoE
|
||||
"openai/gpt-oss-20b", # MXFP4 W4A16
|
||||
]
|
||||
DTYPE = ["bfloat16"]
|
||||
|
||||
|
||||
+14
-1
@@ -3195,6 +3195,7 @@ class CPUQuantMethod(IntEnum):
|
||||
INT8_W8A8 = 1
|
||||
FP8_W8A16 = 2
|
||||
INT4_W4A8 = 3
|
||||
MXFP4 = 4
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
@@ -3213,6 +3214,10 @@ if hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
w1_zero: torch.Tensor | None,
|
||||
w2_zero: torch.Tensor | None,
|
||||
block_size: list[int] | None,
|
||||
w1_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
alpha: float | None,
|
||||
limit: float | None,
|
||||
is_vnni: bool,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
@@ -3231,7 +3236,11 @@ def fused_experts_cpu(
|
||||
w1_zero: torch.Tensor | None,
|
||||
w2_zero: torch.Tensor | None,
|
||||
block_size: list[int] | None,
|
||||
is_vnni: bool,
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
alpha: float | None = None,
|
||||
limit: float | None = None,
|
||||
is_vnni: bool = True,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.fused_experts_cpu(
|
||||
hidden_states,
|
||||
@@ -3246,6 +3255,10 @@ def fused_experts_cpu(
|
||||
w1_zero,
|
||||
w2_zero,
|
||||
block_size,
|
||||
w1_bias,
|
||||
w2_bias,
|
||||
alpha,
|
||||
limit,
|
||||
is_vnni,
|
||||
)
|
||||
|
||||
|
||||
@@ -213,6 +213,10 @@ class SGLFusedMOE:
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
None, # block_size
|
||||
None, # w1_bias
|
||||
None, # w2_bias
|
||||
None, # alpha
|
||||
None, # limit
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU FP8 W8A16 block-quantized fused MoE experts."""
|
||||
"""CPU FP8 W8A16 and MXFP4 W4A16 fused MoE experts."""
|
||||
|
||||
import torch
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -26,14 +27,8 @@ def prepare_fp8_moe_layer_for_cpu(
|
||||
w2: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""VNNI-prepack FP8 MoE weights for CPU kernel."""
|
||||
num_experts = w13.size(0)
|
||||
packed_w13_list = []
|
||||
packed_w2_list = []
|
||||
for i in range(num_experts):
|
||||
packed_w13_list.append(torch.ops._C.convert_weight_packed(w13[i]))
|
||||
packed_w2_list.append(torch.ops._C.convert_weight_packed(w2[i]))
|
||||
packed_w13 = torch.stack(packed_w13_list)
|
||||
packed_w2 = torch.stack(packed_w2_list)
|
||||
packed_w13 = torch.ops._C.convert_weight_packed(w13)
|
||||
packed_w2 = torch.ops._C.convert_weight_packed(w2)
|
||||
return packed_w13, packed_w2
|
||||
|
||||
|
||||
@@ -171,5 +166,161 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
block_shape, # block_size
|
||||
None, # w1_bias
|
||||
None, # w2_bias
|
||||
None, # alpha
|
||||
None, # limit
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
|
||||
def prepare_mxfp4_moe_layer_for_cpu(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""VNNI-prepack MXFP4 MoE weights and repack scales for CPU AMX kernel."""
|
||||
packed_w13 = torch.ops._C.convert_weight_packed(w13)
|
||||
packed_w2 = torch.ops._C.convert_weight_packed(w2)
|
||||
packed_w13_scale = torch.ops._C.convert_scale_packed(w13_scale)
|
||||
packed_w2_scale = torch.ops._C.convert_scale_packed(w2_scale)
|
||||
return packed_w13, packed_w2, packed_w13_scale, packed_w2_scale
|
||||
|
||||
|
||||
class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
|
||||
"""CPU MXFP4 W4A16 monolithic MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cpu()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI)
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kMxfp4Static, None),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return False
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
# Get bias and swiglu params from quant config
|
||||
w1_bias = self.quant_config.w1_bias
|
||||
w2_bias = self.quant_config.w2_bias
|
||||
alpha = getattr(self.quant_config, "gemm1_alpha", None)
|
||||
limit = getattr(self.quant_config, "gemm1_clamp_limit", None)
|
||||
|
||||
return fused_experts_cpu(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
CPUQuantMethod.MXFP4, # moe_comp_method
|
||||
self.w1_scale, # w1_scale
|
||||
self.w2_scale, # w2_scale
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
None, # block_size
|
||||
w1_bias,
|
||||
w2_bias,
|
||||
alpha,
|
||||
limit,
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
@@ -79,6 +79,8 @@ class Mxfp4MoeBackend(Enum):
|
||||
TRITON_UNFUSED = "TRITON_UNFUSED"
|
||||
# XPU
|
||||
XPU = "XPU"
|
||||
# CPU
|
||||
CPU = "CPU"
|
||||
# Emulation
|
||||
EMULATION = "EMULATION"
|
||||
# Humming
|
||||
@@ -198,6 +200,11 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [XPUExpertsMXFp4]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import CPUExpertsMxfp4
|
||||
|
||||
return [CPUExpertsMxfp4]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.EMULATION:
|
||||
from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import (
|
||||
OCP_MXQuantizationEmulationTritonExperts,
|
||||
@@ -237,6 +244,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
|
||||
],
|
||||
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
|
||||
"xpu": [Mxfp4MoeBackend.XPU],
|
||||
"cpu": [Mxfp4MoeBackend.CPU],
|
||||
"emulation": [Mxfp4MoeBackend.EMULATION],
|
||||
}
|
||||
if backends := mapping.get(runner_backend):
|
||||
@@ -535,6 +543,17 @@ def select_mxfp4_moe_backend(
|
||||
activation_format,
|
||||
)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
backend = Mxfp4MoeBackend.CPU
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return _return_or_raise(
|
||||
Mxfp4MoeBackend.CPU,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
None,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
if current_platform.is_cuda() or current_platform.is_rocm():
|
||||
raise NotImplementedError(
|
||||
"No MXFP4 MoE backend supports the deployment configuration."
|
||||
@@ -639,6 +658,10 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
elif current_platform.is_rocm():
|
||||
intermediate_size = round_up(intermediate_size, 256)
|
||||
hidden_size = round_up(hidden_size, 256)
|
||||
elif backend == Mxfp4MoeBackend.CPU:
|
||||
# CPU AMX kernel uses BLOCK_N=32, align to 32
|
||||
intermediate_size = round_up(intermediate_size, 32)
|
||||
hidden_size = round_up(hidden_size, 32)
|
||||
else:
|
||||
intermediate_size = round_up(intermediate_size, 64)
|
||||
return hidden_size, intermediate_size
|
||||
@@ -1098,6 +1121,31 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_mxfp4_moe_layer_for_cpu,
|
||||
)
|
||||
|
||||
packed_w13, packed_w2, packed_w13_scale, packed_w2_scale = (
|
||||
prepare_mxfp4_moe_layer_for_cpu(
|
||||
w13_weight.data,
|
||||
w2_weight.data,
|
||||
w13_weight_scale.data,
|
||||
w2_weight_scale.data,
|
||||
)
|
||||
)
|
||||
if w13_bias is not None:
|
||||
w13_bias = w13_bias.data.to(torch.float32)
|
||||
if w2_bias is not None:
|
||||
w2_bias = w2_bias.data.to(torch.float32)
|
||||
return (
|
||||
packed_w13,
|
||||
packed_w2,
|
||||
packed_w13_scale,
|
||||
packed_w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.EMULATION:
|
||||
# No additional transformation needed for emulation backend,
|
||||
# weights are dequantized on the fly in the experts class.
|
||||
@@ -1501,6 +1549,7 @@ def make_mxfp4_moe_quant_config(
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.CPU,
|
||||
):
|
||||
return mxfp4_w4a16_moe_quant_config(
|
||||
w1_bias=w1_bias,
|
||||
|
||||
Reference in New Issue
Block a user