From ca29cd48ba05ac9ae1c9fa29271e9f899bc4ce56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elvir=20Crn=C4=8Devi=C4=87?= Date: Tue, 25 Aug 2026 16:21:00 +0200 Subject: [PATCH] [Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Elvir Crncevic Signed-off-by: Tyler Michael Smith Signed-off-by: Elvir Crnčević Co-authored-by: Claude Opus 4.8 Co-authored-by: Tyler Michael Smith Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- csrc/libtorch_stable/activation_kernels.cu | 415 +++++++++++++++++- csrc/libtorch_stable/ops.h | 12 +- csrc/libtorch_stable/torch_bindings.cpp | 11 +- ...0B-A3B-MXFP4A16-humming-act-fp8-block.yaml | 11 + .../gsm8k/configs/humming/config-act-fp8.txt | 1 + tests/kernels/moe/test_moe.py | 1 + tests/kernels/moe/test_moe_fused_mul_sum.py | 406 +++++++++++++++++ tests/kernels/moe/test_situ_mul_fp8_quant.py | 224 ++++++++++ .../test_humming_mxfp4_block_fp8.py | 122 +++++ .../layers/fused_moe/activation.py | 44 ++ .../layers/fused_moe/deep_gemm_utils.py | 4 +- .../fused_moe/experts/fused_batched_moe.py | 1 + .../fused_moe/experts/fused_humming_moe.py | 255 +++++++++-- .../layers/fused_moe/modular_kernel.py | 5 +- .../layers/fused_moe/moe_fused_mul_sum.py | 204 ++++----- .../fused_moe/prepare_finalize/deepep_v2.py | 14 +- .../quantization/utils/humming_utils.py | 35 +- 17 files changed, 1594 insertions(+), 171 deletions(-) create mode 100644 tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8-block.yaml create mode 100644 tests/kernels/moe/test_moe_fused_mul_sum.py create mode 100644 tests/kernels/moe/test_situ_mul_fp8_quant.py create mode 100644 tests/quantization/test_humming_mxfp4_block_fp8.py diff --git a/csrc/libtorch_stable/activation_kernels.cu b/csrc/libtorch_stable/activation_kernels.cu index aa2c1b6b92..56d0058dfc 100644 --- a/csrc/libtorch_stable/activation_kernels.cu +++ b/csrc/libtorch_stable/activation_kernels.cu @@ -1,9 +1,17 @@ #include +#include #include +#include + +#ifndef USE_ROCM + #include +#endif #include +#include #include "../cuda_compat.h" +#include "async_util.cuh" #include "cuda_vec_utils.cuh" #include "dispatch_utils.h" #include "torch_utils.h" @@ -471,15 +479,31 @@ __global__ void swigluoai_and_mul_kernel( // gate_out = beta * tanh(gate / beta) * sigmoid(gate) // up_out = (linear_beta > 0) ? linear_beta * tanh(up / linear_beta) : up // out = gate_out * up_out -// Compute is done in fp32 and written straight to `out` -- no intermediate -// tensors and no full-tensor fp32 upcast (the pure-torch forward_native -// allocated ~8 fp32 temporaries per call, which blows up MoE profiling). +__device__ __forceinline__ float situ_tanh(float x) { return tanhf(x); } + +// Kimi-K3 SITU params; baked into the fused LDG kernel to fold at compile time. +static constexpr float SITU_BETA = 4.0f; +static constexpr float SITU_LINEAR_BETA = 25.0f; +__device__ __forceinline__ float situ_activation(float g, float u, float beta, + float linear_beta, + bool clamp_up, float inv_beta, + float inv_linear_beta) { + // sigmoid(g) == (1 + tanh(g/2)) / 2. + const float gate_out = + (0.5f * beta) * situ_tanh(g * inv_beta) * (1.0f + situ_tanh(g * 0.5f)); + const float up_out = + clamp_up ? linear_beta * situ_tanh(u * inv_linear_beta) : u; + return gate_out * up_out; +} + template __global__ void situ_and_mul_kernel( scalar_t* __restrict__ out, // [..., d] const scalar_t* __restrict__ input, // [..., 2, d] - const int d, const float beta, const float linear_beta) { + const int d, const float beta, const float linear_beta, + const int64_t* __restrict__ valid_rows_ptr) { const int64_t row = blockIdx.x; + if (valid_rows_ptr != nullptr && row >= *valid_rows_ptr) return; const scalar_t* gate_ptr = input + row * 2 * d; const scalar_t* up_ptr = gate_ptr + d; scalar_t* out_ptr = out + row * d; @@ -489,10 +513,258 @@ __global__ void situ_and_mul_kernel( for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) { const float g = (float)VLLM_LDG(&gate_ptr[idx]); const float u = (float)VLLM_LDG(&up_ptr[idx]); - const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g)); - const float up_out = - clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u; - out_ptr[idx] = (scalar_t)(gate_out * up_out); + out_ptr[idx] = (scalar_t)situ_activation(g, u, beta, linear_beta, clamp_up, + inv_beta, inv_linear_beta); + } +} + +// Match Humming's hardware FP8 conversion. c10::Float8_e4m3fn's software cast +// can round ties differently from `cvt.rn.satfinite.e4m3.f32`. +__device__ __forceinline__ c10::Float8_e4m3fn cvt_fp8_hw(float x) { +#if !defined(USE_ROCM) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + return c10::Float8_e4m3fn(__nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3), + c10::Float8_e4m3fn::from_bits()); +#else + return static_cast(x); +#endif +} + +template +__device__ __forceinline__ fp8_type quant_to_fp8(float val, float inv_scale, + float fp8_max) { + float x = val * inv_scale; + x = fmaxf(-fp8_max, fminf(x, fp8_max)); + if constexpr (std::is_same_v) { + return cvt_fp8_hw(x); + } else { + return static_cast(x); + } +} + +__device__ __forceinline__ float warp_reduce_max(float v) { +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { + v = fmaxf(v, VLLM_SHFL_XOR_SYNC(v, offset)); + } + return v; +} + +__device__ __forceinline__ float warp_reduce_absmax(float v) { + return warp_reduce_max(v); +} + +template +__global__ void situ_and_mul_quant_group_pipelined_kernel( + fp8_type* __restrict__ out, float* __restrict__ scale_out, + const scalar_t* __restrict__ input, const int64_t num_rows, + const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) { + const int tid = threadIdx.x; + const int warp_id = tid / WARP_SIZE; + const int lane_id = tid % WARP_SIZE; + static constexpr int NUM_WARPS = THREADS / WARP_SIZE; + static constexpr int NUM_GROUPS = D / GROUP_SIZE; + const int64_t row_bound = + valid_rows_ptr != nullptr + ? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows)) + : num_rows; + + static_assert(NUM_GROUPS % 4 == 0, + "float4 scale fill needs NUM_GROUPS % 4 == 0"); + static constexpr int NG4 = NUM_GROUPS / 4; + const int64_t pad_start4 = row_bound * NG4; + const int64_t pad_end4 = num_rows * NG4; + float4* scale4 = reinterpret_cast(scale_out); + constexpr float4 ones{1.0f, 1.0f, 1.0f, 1.0f}; + for (int64_t i = pad_start4 + (int64_t)blockIdx.x * THREADS + tid; + i < pad_end4; i += (int64_t)GRID_DIM * THREADS) { + scale4[i] = ones; + } + + if constexpr (sizeof(scalar_t) == 2) { + static constexpr int LD_ELTS = 16 / sizeof(scalar_t); + static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE; + static constexpr int STAGE_ELTS = 2 * GROUP_SIZE; + + extern __shared__ __align__(16) unsigned char smem_raw[]; + scalar_t* warp_smem = reinterpret_cast(smem_raw) + + (size_t)warp_id * NUM_STAGES * STAGE_ELTS; + + static constexpr float beta = SITU_BETA; + static constexpr float linear_beta = SITU_LINEAR_BETA; + static constexpr bool clamp_up = linear_beta > 0.0f; + static constexpr float inv_beta = 1.0f / beta; + static constexpr float inv_linear_beta = + clamp_up ? 1.0f / linear_beta : 0.0f; + static constexpr float FP8_MAX = + std::is_same_v ? 448.0f : 224.0f; + static constexpr float inv_fp8_max = 1.0f / FP8_MAX; + + static_assert( + NUM_GROUPS % NUM_WARPS == 0, + "constexpr num_iters requires groups evenly split across warps"); + static constexpr int num_iters = NUM_GROUPS / NUM_WARPS; + + const bool up_half = lane_id >= WARP_SIZE / 2; + const int lane_l = up_half ? lane_id - WARP_SIZE / 2 : lane_id; + const int lane_src_off = (up_half ? D : 0) + lane_l * LD_ELTS; + const int lane_dst_off = (up_half ? GROUP_SIZE : 0) + lane_l * LD_ELTS; + static constexpr int warp_stride = NUM_WARPS * GROUP_SIZE; + + const scalar_t* src_ptr = input + warp_id * GROUP_SIZE + lane_src_off; + static constexpr int src_row_stride = GRID_DIM * 2 * D; + const scalar_t* row_src = src_ptr + blockIdx.x * 2 * D; + + uint32_t* out_ptr = + reinterpret_cast(out + warp_id * GROUP_SIZE) + lane_id; + static constexpr int out_row_stride = GRID_DIM * D / 4; + uint32_t* row_out = out_ptr + blockIdx.x * (D / 4); + float* scale_ptr = scale_out + warp_id; + static constexpr int scale_row_stride = GRID_DIM * NUM_GROUPS; + float* row_scale = scale_ptr + blockIdx.x * NUM_GROUPS; + +#pragma unroll 1 + for (int64_t row = blockIdx.x; row < row_bound; row += GRID_DIM, + row_src += src_row_stride, row_out += out_row_stride, + row_scale += scale_row_stride) { + const scalar_t* src = row_src; + + auto issue_load = [&](int it, int slot) { + if (it < num_iters) { + cuda_async::cp_async_shared_global_16_cg( + warp_smem + (size_t)slot * STAGE_ELTS + lane_dst_off, src); + src += warp_stride; + } + cuda_async::cp_async_commit_group(); + }; + + int load_slot = 0; + auto bump = [](int s) { return s + 1 == NUM_STAGES ? 0 : s + 1; }; +#pragma unroll + for (int s = 0; s < NUM_STAGES - 1; s++) { + issue_load(s, load_slot); + load_slot = bump(load_slot); + } + + uint32_t* out_st = row_out; + float* scale_st = row_scale; + + int comp_slot = 0; + for (int it = 0; it < num_iters; it++) { + issue_load(it + NUM_STAGES - 1, load_slot); + load_slot = bump(load_slot); + cuda_async::cp_async_wait_group(); + + const scalar_t* stage = warp_smem + (size_t)comp_slot * STAGE_ELTS; + comp_slot = bump(comp_slot); + static_assert(ELTS_PER_LANE == 4, + "expects GROUP_SIZE == 4 * WARP_SIZE"); + union V { + float2 f2; + scalar_t s[ELTS_PER_LANE]; + }; + const float2* gate2 = reinterpret_cast(stage); + const float2* up2 = reinterpret_cast(stage + GROUP_SIZE); + const V gv{gate2[lane_id]}; + const V uv{up2[lane_id]}; + const scalar_t* gs = gv.s; + const scalar_t* us = uv.s; + + float acts[ELTS_PER_LANE]; + float thread_max = 0.0f; +#pragma unroll + for (int e = 0; e < ELTS_PER_LANE; e++) { + acts[e] = (float)(scalar_t)situ_activation( + (float)gs[e], (float)us[e], beta, linear_beta, clamp_up, inv_beta, + inv_linear_beta); + thread_max = fmaxf(thread_max, fabsf(acts[e])); + } + const float absmax = fmaxf(warp_reduce_absmax(thread_max), 1e-30f); + const float scale = absmax * inv_fp8_max; + if (lane_id == 0) *scale_st = scale; + scale_st += NUM_WARPS; + const float inv_scale = __fdividef(1.0f, scale); + + union O { + uint32_t u; + fp8_type f[ELTS_PER_LANE]; + }; + O o; +#pragma unroll + for (int e = 0; e < ELTS_PER_LANE; e++) { + o.f[e] = quant_to_fp8(acts[e], inv_scale, FP8_MAX); + } + out_st[0] = o.u; + out_st += NUM_WARPS * GROUP_SIZE / 4; + } + cuda_async::cp_async_wait_group<0>(); + } + } +} + +// Scalar fallback for the group path (odd d or the fp32 dispatch branch). +template +__global__ void situ_and_mul_quant_group_scalar_kernel( + fp8_type* __restrict__ out, // [num_tokens, d] + float* __restrict__ scale_out, // [num_tokens, num_groups] + const scalar_t* __restrict__ input, // [num_tokens, 2, d] + const int d, const int num_groups, const float beta, + const float linear_beta, const int64_t num_rows, + const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) { + static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE; + const int tid = threadIdx.x; + const int warp_id = tid / WARP_SIZE; + const int lane_id = tid % WARP_SIZE; + const int num_warps = blockDim.x / WARP_SIZE; + const int64_t row_bound = + valid_rows_ptr != nullptr + ? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows)) + : num_rows; + + const bool clamp_up = linear_beta > 0.0f; + const float inv_beta = __fdividef(1.0f, beta); + const float inv_linear_beta = clamp_up ? __fdividef(1.0f, linear_beta) : 0.0f; + static constexpr float FP8_MAX = + std::is_same_v ? 448.0f : 224.0f; + static constexpr float inv_fp8_max = 1.0f / FP8_MAX; + + for (int64_t row = blockIdx.x; row < row_bound; row += gridDim.x) { + const scalar_t* gate_ptr = input + row * 2 * (int64_t)d; + const scalar_t* up_ptr = gate_ptr + d; + fp8_type* out_ptr = out + row * (int64_t)d; + float* scale_row = scale_out + row * (int64_t)num_groups; + + for (int g = warp_id; g < num_groups; g += num_warps) { + const int base = g * GROUP_SIZE; + float acts[ELTS_PER_LANE]; + float thread_max = 0.0f; +#pragma unroll + for (int e = 0; e < ELTS_PER_LANE; e++) { + const int idx = base + e * WARP_SIZE + lane_id; + const float gv = (float)VLLM_LDG(&gate_ptr[idx]); + const float uv = (float)VLLM_LDG(&up_ptr[idx]); + acts[e] = (float)(scalar_t)situ_activation( + gv, uv, beta, linear_beta, clamp_up, inv_beta, inv_linear_beta); + thread_max = fmaxf(thread_max, fabsf(acts[e])); + } + const float absmax = fmaxf(warp_reduce_max(thread_max), 1e-30f); + const float scale = absmax * inv_fp8_max; + if (lane_id == 0) scale_row[g] = scale; + const float inv_scale = __fdividef(1.0f, scale); +#pragma unroll + for (int e = 0; e < ELTS_PER_LANE; e++) { + const int idx = base + e * WARP_SIZE + lane_id; + out_ptr[idx] = quant_to_fp8(acts[e], inv_scale, FP8_MAX); + } + } + } + + // Fill skipped padding-row scales with 1 so they don't feed NaN into w2. + const int64_t pad_start = row_bound * (int64_t)num_groups; + const int64_t pad_end = num_rows * (int64_t)num_groups; + for (int64_t i = pad_start + (int64_t)blockIdx.x * blockDim.x + tid; + i < pad_end; i += (int64_t)gridDim.x * blockDim.x) { + scale_out[i] = 1.0f; } } @@ -519,10 +791,8 @@ __global__ void masked_situ_and_mul_kernel( scalar_t* out_ptr = out + row * d; const float g = (float)VLLM_LDG(&gate_ptr[idx]); const float u = (float)VLLM_LDG(&up_ptr[idx]); - const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g)); - const float up_out = - clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u; - out_ptr[idx] = (scalar_t)(gate_out * up_out); + out_ptr[idx] = (scalar_t)situ_activation(g, u, beta, linear_beta, clamp_up, + inv_beta, inv_linear_beta); } } @@ -620,7 +890,8 @@ void swigluoai_and_mul(torch::stable::Tensor& out, // [..., d] // through), matching SituAndMul(linear_beta=None) on the Python side. void situ_and_mul(torch::stable::Tensor& out, // [..., d] torch::stable::Tensor& input, // [..., 2 * d] - double beta, double linear_beta) { + double beta, double linear_beta, + std::optional valid_rows) { int d = input.size(-1) / 2; int64_t num_tokens = input.numel() / input.size(-1); if (num_tokens == 0) { @@ -628,6 +899,14 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d] } dim3 grid(num_tokens); dim3 block(std::min(d, 1024)); + const int64_t* valid_rows_ptr = nullptr; + if (valid_rows.has_value()) { + STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() == + input.get_device_index(), + "situ_and_mul: valid_rows must be a CUDA tensor on the " + "same device as input"); + valid_rows_ptr = valid_rows->const_data_ptr(); + } const torch::stable::accelerator::DeviceGuard device_guard( input.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(); @@ -635,7 +914,7 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d] input.scalar_type(), "situ_and_mul_kernel", [&] { vllm::situ_and_mul_kernel<<>>( out.mutable_data_ptr(), input.const_data_ptr(), - d, (float)beta, (float)linear_beta); + d, (float)beta, (float)linear_beta, valid_rows_ptr); }); } @@ -663,6 +942,114 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d] (float)beta, (float)linear_beta); }); } + +// Fused SITU activation and block-FP8 quantization for Humming w2. +// `valid_rows` is the int32 DeepEP token count (psum[-1]); rows = it * `topk`, +// so padding is excluded on-device and skipped rows receive scale 1. +void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8) + torch::stable::Tensor& scale, // [..., 1 or d/128] + torch::stable::Tensor& input, // [..., 2 * d] + double beta, double linear_beta, int64_t group_size, + std::optional valid_rows, + int64_t topk) { + STD_TORCH_CHECK( + out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn || + out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fnuz, + "situ_and_mul_quant output must be FP8 e4m3"); + STD_TORCH_CHECK( + input.scalar_type() == torch::headeronly::ScalarType::Half || + input.scalar_type() == torch::headeronly::ScalarType::BFloat16, + "situ_and_mul_quant input must be FP16 or BF16"); + STD_TORCH_CHECK(scale.scalar_type() == torch::headeronly::ScalarType::Float, + "situ_and_mul_quant scale must be float32"); + STD_TORCH_CHECK(input.is_cuda() && out.is_cuda() && scale.is_cuda() && + out.get_device_index() == input.get_device_index() && + scale.get_device_index() == input.get_device_index(), + "situ_and_mul_quant: input, out and scale must be CUDA " + "tensors on the same device"); + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + if (num_tokens == 0) { + return; + } + STD_TORCH_CHECK(out.size(-1) == d && out.numel() == num_tokens * (int64_t)d, + "situ_and_mul_quant: out shape must be [num_tokens, d]"); + const int32_t* valid_rows_ptr = nullptr; + if (valid_rows.has_value()) { + STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() == + input.get_device_index(), + "situ_and_mul_quant: valid_rows must be a CUDA tensor on " + "the same device as input"); + STD_TORCH_CHECK( + valid_rows->scalar_type() == torch::headeronly::ScalarType::Int, + "situ_and_mul_quant: valid_rows must be int32 (psum count)"); + valid_rows_ptr = valid_rows->const_data_ptr(); + } + const torch::stable::accelerator::DeviceGuard device_guard( + input.get_device_index()); + const cudaStream_t stream = get_current_cuda_stream(); + static constexpr int BLOCKS_PER_SM = 8; // matches kernel __launch_bounds__ + static constexpr int SM_COUNT = 132; // H200 (GH100, 132 SMs) + static constexpr int GRID_DIM = SM_COUNT * BLOCKS_PER_SM; + + STD_TORCH_CHECK(group_size == 128, + "situ_and_mul_quant: only group_size 128 (block-FP8) " + "supported, got ", + group_size); + STD_TORCH_CHECK(d % group_size == 0, "situ_and_mul_quant: d (", d, + ") must be divisible by group_size ", group_size); + { + const int num_groups = d / (int)group_size; + STD_TORCH_CHECK(scale.size(-1) == num_groups && + scale.numel() == num_tokens * (int64_t)num_groups, + "situ_and_mul_quant: scale shape must be " + "[num_tokens, d/group_size]"); + dim3 grid(GRID_DIM); + VLLM_STABLE_DISPATCH_FLOATING_TYPES( + input.scalar_type(), "situ_and_mul_quant_group_kernel", [&] { + VLLM_STABLE_DISPATCH_FP8_TYPES( + out.scalar_type(), "situ_and_mul_quant_group_kernel_fp8", [&] { +#ifndef USE_ROCM + // The pipelined kernel's float2-per-lane geometry assumes a + // 32-lane warp (GROUP_SIZE == 4 * WARP_SIZE); on HIP (64-lane) + // fall back to the WARP_SIZE-generic scalar kernel. + constexpr int THREADS = 256; + constexpr int SITU_D = 3072; // Kimi-K3 fused w2 input dim + if constexpr (sizeof(scalar_t) == 2) { + if (d == SITU_D && (float)beta == vllm::SITU_BETA && + (float)linear_beta == vllm::SITU_LINEAR_BETA) { + constexpr int D = SITU_D; + constexpr int GROUP_STAGES = 4; + constexpr int NUM_WARPS = THREADS / 32; + constexpr int STAGE_ELTS = 2 * 128; + dim3 block(THREADS); + size_t smem_bytes = (size_t)NUM_WARPS * GROUP_STAGES * + STAGE_ELTS * sizeof(scalar_t); + vllm::situ_and_mul_quant_group_pipelined_kernel< + scalar_t, fp8_t, THREADS, GROUP_STAGES, 128, GRID_DIM, + D><<>>( + out.mutable_data_ptr(), + scale.mutable_data_ptr(), + input.const_data_ptr(), num_tokens, + valid_rows_ptr, topk); + return; + } + } +#endif + const int num_warps = std::min(num_groups, 1024 / WARP_SIZE); + dim3 block(num_warps * WARP_SIZE); + vllm::situ_and_mul_quant_group_scalar_kernel + <<>>( + out.mutable_data_ptr(), + scale.mutable_data_ptr(), + input.const_data_ptr(), d, num_groups, + (float)beta, (float)linear_beta, num_tokens, + valid_rows_ptr, topk); + }); + }); + } +} namespace vllm { // Element-wise activation kernel template. diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 682ede771a..b0c8d387ba 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -526,8 +526,16 @@ void fatrelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, double threshold); void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, double alpha = 1.702, double limit = 7.0); -void situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, - double beta = 1.0, double linear_beta = -1.0); +void situ_and_mul( + torch::stable::Tensor& out, torch::stable::Tensor& input, double beta = 1.0, + double linear_beta = -1.0, + std::optional valid_rows = std::nullopt); +void situ_and_mul_quant( + torch::stable::Tensor& out, torch::stable::Tensor& scale, + torch::stable::Tensor& input, double beta = 1.0, double linear_beta = -1.0, + int64_t group_size = 0, + std::optional valid_rows = std::nullopt, + int64_t topk = 1); void masked_situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, const torch::stable::Tensor& expert_num_tokens, diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 3e92e7f911..c9bcba1549 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -611,7 +611,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { // SituGLU implementation used in Kimi models. ops.def( "situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float " - "linear_beta=-1.0) -> ()"); + "linear_beta=-1.0, Tensor? valid_rows=None) -> ()"); + // Fused SituGLU activation + dynamic FP8 quantization for the Humming w2 path + // (writes the fp8 down input and its float32 scale). group_size=0 -> + // per-token scale [.., 1]; group_size=128 -> k-major block-FP8 scale [.., + // d/128]. + ops.def( + "situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, " + "float beta=1.0, float linear_beta=-1.0, int group_size=0, " + "Tensor? valid_rows=None, int topk=1) -> ()"); ops.def( "masked_situ_and_mul(Tensor! out, Tensor input, Tensor " "expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()"); @@ -830,6 +838,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) { ops.impl("fatrelu_and_mul", TORCH_BOX(&fatrelu_and_mul)); ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul)); ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul)); + ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant)); ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul)); ops.impl("gelu_new", TORCH_BOX(&gelu_new)); ops.impl("gelu_fast", TORCH_BOX(&gelu_fast)); diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8-block.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8-block.yaml new file mode 100644 index 0000000000..c6ab8d393e --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8-block.yaml @@ -0,0 +1,11 @@ +model_name: "nm-testing/Qwen3-30B-A3B-MXFP4A16" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --quantization humming + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3","group_size":128}' diff --git a/tests/evals/gsm8k/configs/humming/config-act-fp8.txt b/tests/evals/gsm8k/configs/humming/config-act-fp8.txt index 42ff6be00e..4c85940495 100644 --- a/tests/evals/gsm8k/configs/humming/config-act-fp8.txt +++ b/tests/evals/gsm8k/configs/humming/config-act-fp8.txt @@ -1,5 +1,6 @@ gpt-oss-20b-humming-act-fp8.yaml Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml +Qwen3-30B-A3B-MXFP4A16-humming-act-fp8-block.yaml Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml Qwen3-0.6B-MXFP8-humming-act-fp8.yaml Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml diff --git a/tests/kernels/moe/test_moe.py b/tests/kernels/moe/test_moe.py index ac73ca9416..26e0370a3e 100644 --- a/tests/kernels/moe/test_moe.py +++ b/tests/kernels/moe/test_moe.py @@ -1408,6 +1408,7 @@ def test_humming_delegates_to_instance_activation(): activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE, input=input, output=output, + valid_rows=None, ) diff --git a/tests/kernels/moe/test_moe_fused_mul_sum.py b/tests/kernels/moe/test_moe_fused_mul_sum.py new file mode 100644 index 0000000000..5affc6ea16 --- /dev/null +++ b/tests/kernels/moe/test_moe_fused_mul_sum.py @@ -0,0 +1,406 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the fused MoE topk weight-and-reduce kernel. + +Run `pytest tests/kernels/moe/test_moe_fused_mul_sum.py`. +""" + +import pytest +import torch + +from vllm.model_executor.layers.fused_moe.moe_fused_mul_sum import ( + moe_fused_mul_sum, +) +from vllm.utils.torch_utils import set_random_seed + +NUM_TOKENS = [1, 17, 256] +TOP_KS = [2, 8] +HIDDEN_SIZE = 512 +# One BLOCK_K tile, a partial trailing tile, and Kimi-K3's real dims +# (moe_intermediate_size 3072, hidden_size 7168) -- exercises the per-tile loop +# and its tail mask (BLOCK_K caps at 512 for <=2-byte dtypes). +HIDDEN_SIZES = [512, 1000, 3072, 7168] +DTYPES = [torch.bfloat16, torch.float32] +NUM_EXPERTS = 16 + + +def _reference( + inputs: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + expert_map: torch.Tensor | None, +) -> torch.Tensor: + valid = topk_ids >= 0 + if expert_map is not None: + valid &= expert_map[topk_ids.clamp(min=0)] >= 0 + weights = torch.where(valid, topk_weights, torch.zeros_like(topk_weights)) + return (inputs.float() * weights.float().unsqueeze(-1)).sum(dim=1) + + +@pytest.mark.parametrize("num_tokens", NUM_TOKENS) +@pytest.mark.parametrize("top_k", TOP_KS) +@pytest.mark.parametrize("use_expert_map", [True, False]) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_invalid_slots_excluded(num_tokens: int, top_k: int, use_expert_map: bool): + """Slots marked -1 must not contribute, and must not be dereferenced. + + The expert GEMM never writes rows for topk slots whose expert id is -1 + (a non-local expert under EP, or an all2all padding row), so those rows + hold stale workspace data. Filling them with NaN here means any failure to + mask them shows up as NaN in the output rather than a small numeric drift. + Indexing `expert_map` with -1 is also an out-of-bounds read. + """ + set_random_seed(0) + device = "cuda" + + inputs = torch.randn( + num_tokens, top_k, HIDDEN_SIZE, dtype=torch.bfloat16, device=device + ) + topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device) + topk_ids = torch.randint( + 0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device + ) + + # Mark roughly half the slots invalid and poison the rows behind them. + invalid = torch.rand(num_tokens, top_k, device=device) < 0.5 + topk_ids[invalid] = -1 + inputs[invalid] = float("nan") + + expert_map = None + if use_expert_map: + # Second half of the experts is non-local, so valid ids can still map + # to -1 -- exercising the second masking condition. + expert_map = torch.full((NUM_EXPERTS,), -1, dtype=torch.int32, device=device) + local = NUM_EXPERTS // 2 + expert_map[:local] = torch.arange(local, dtype=torch.int32, device=device) + inputs[(topk_ids >= 0) & (expert_map[topk_ids.clamp(min=0)] < 0)] = float("nan") + + # The _reference implementation zeros out outputs. We need to match that here, + # otherwise the test fails. + outputs = torch.zeros(num_tokens, HIDDEN_SIZE, dtype=torch.bfloat16, device=device) + out = moe_fused_mul_sum( + inputs=inputs, + topk_weights=topk_weights, + outputs=outputs, + topk_ids=topk_ids, + expert_map=expert_map, + ) + + assert not out.isnan().any(), "invalid slots leaked into the reduction" + ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map) + torch.testing.assert_close(out.float(), ref, atol=1e-2, rtol=1e-2) + + +def _local_expert_map(device: str) -> torch.Tensor: + """First half of the experts is local, second half maps to -1 (non-local).""" + expert_map = torch.full((NUM_EXPERTS,), -1, dtype=torch.int32, device=device) + local = NUM_EXPERTS // 2 + expert_map[:local] = torch.arange(local, dtype=torch.int32, device=device) + return expert_map + + +SENTINEL = 42.0 # exact in bf16; marks output the kernel must leave untouched + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_nonlocal_row_is_zeroed(): + """A real row whose positive ids all map to non-local experts is written 0. + + Such a row is not padding (its ids are >= 0, so row_valid > 0 and the block + is not early-returned). Every slot is then masked out by expert_map, so the + accumulator stays 0 and 0 is stored -- the row must not be left holding stale + buffer data, because downstream reduction treats it as a real token. + """ + set_random_seed(0) + device = "cuda" + num_tokens, top_k = 17, 8 + local = NUM_EXPERTS // 2 + expert_map = _local_expert_map(device) + + # Poison inputs so any failure to mask the non-local slots surfaces as NaN. + inputs = torch.full( + (num_tokens, top_k, HIDDEN_SIZE), + float("nan"), + dtype=torch.bfloat16, + device=device, + ) + topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device) + # Every id lands in the non-local half [local, NUM_EXPERTS). + topk_ids = torch.randint( + local, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device + ) + + outputs = torch.full( + (num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device + ) + out = moe_fused_mul_sum( + inputs=inputs, + topk_weights=topk_weights, + outputs=outputs, + topk_ids=topk_ids, + expert_map=expert_map, + ) + + assert not out.isnan().any(), "non-local row leaked NaN into the output" + torch.testing.assert_close(out.float(), torch.zeros_like(out.float())) + + +@pytest.mark.parametrize("use_expert_map", [True, False]) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_padding_row_left_untouched(use_expert_map: bool): + """An all -1 padding row is skipped: its output row is left untouched. + + This is the CUDA-graph decode contract -- padding rows past num_recv are + never read downstream, so the kernel elides them (whole-padding blocks + early-return) rather than zeroing. The skip is gated on topk_ids alone, so + it must hold whether or not expert_map is passed (the humming path passes + None); we pin both. + """ + set_random_seed(0) + device = "cuda" + num_tokens, top_k = 17, 8 + expert_map = _local_expert_map(device) if use_expert_map else None + + inputs = torch.full( + (num_tokens, top_k, HIDDEN_SIZE), + float("nan"), + dtype=torch.bfloat16, + device=device, + ) + topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device) + topk_ids = torch.full((num_tokens, top_k), -1, dtype=torch.int32, device=device) + + outputs = torch.full( + (num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device + ) + out = moe_fused_mul_sum( + inputs=inputs, + topk_weights=topk_weights, + outputs=outputs, + topk_ids=topk_ids, + expert_map=expert_map, + ) + + assert torch.all(out == SENTINEL), "padding row output was modified" + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_nonlocal_and_padding_adjacent(): + """A non-local real row next to an all -1 padding row. + + Each CTA owns one token row: the non-local row must be zeroed (its slots are + all masked by expert_map) while the padding row beside it is left untouched. + Guards against a change that lets one row's fate leak into its neighbour. + """ + set_random_seed(0) + device = "cuda" + num_tokens, top_k = 17, 8 + local = NUM_EXPERTS // 2 + expert_map = _local_expert_map(device) + + inputs = torch.full( + (num_tokens, top_k, HIDDEN_SIZE), + float("nan"), + dtype=torch.bfloat16, + device=device, + ) + topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device) + # All rows are non-local real rows except row 1, an all -1 padding row. + topk_ids = torch.randint( + local, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device + ) + topk_ids[1] = -1 + + outputs = torch.full( + (num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device + ) + out = moe_fused_mul_sum( + inputs=inputs, + topk_weights=topk_weights, + outputs=outputs, + topk_ids=topk_ids, + expert_map=expert_map, + ) + + padding = torch.zeros(num_tokens, dtype=torch.bool, device=device) + padding[1] = True + assert torch.all(out[1] == SENTINEL), "padding row beside a real row was modified" + written = out[~padding].float() + assert not written.isnan().any(), "non-local row leaked NaN into the output" + torch.testing.assert_close(written, torch.zeros_like(written)) + + +@pytest.mark.parametrize("num_tokens", NUM_TOKENS) +@pytest.mark.parametrize("top_k", TOP_KS) +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_weighted_sum_matches_reference( + num_tokens: int, top_k: int, hidden_size: int, dtype: torch.dtype +): + """Weighted sum over live slots matches the reference across hidden tiles. + + Covers the deployed humming path (globalized ids, expert_map=None): a random + -1 mix with NaN-poisoned dead slots, over hidden sizes that span a single + tile, a partial trailing tile, and many tiles, in both bf16 and fp32. + """ + set_random_seed(0) + device = "cuda" + + inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device) + topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device) + topk_ids = torch.randint( + 0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device + ) + invalid = torch.rand(num_tokens, top_k, device=device) < 0.5 + topk_ids[invalid] = -1 + inputs[invalid] = float("nan") + + # All-padding rows early-return untouched, so pre-zero to match the reference. + outputs = torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device) + out = moe_fused_mul_sum( + inputs=inputs, + topk_weights=topk_weights, + outputs=outputs, + topk_ids=topk_ids, + expert_map=None, + ) + + assert not out.isnan().any(), "invalid slots leaked into the reduction" + ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map=None) + atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4) + torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol) + + +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_no_topk_ids_sums_all_slots(dtype: torch.dtype): + """With topk_ids=None every slot is live: plain weighted sum over top_k.""" + set_random_seed(0) + device = "cuda" + num_tokens, top_k, hidden_size = 17, 8, 1000 + + inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device) + topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device) + out = moe_fused_mul_sum(inputs=inputs, topk_weights=topk_weights) + + ref = (inputs.float() * topk_weights.float().unsqueeze(-1)).sum(dim=1) + atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4) + torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol) + + +@pytest.mark.parametrize("hidden_size", [512, 7168]) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_expert_map_is_required_for_nonlocal_ids(hidden_size: int, dtype: torch.dtype): + """expert_map is load-bearing, not redundant: dropping it leaks non-local ids. + + A positive id whose expert is non-local marks a slot the GEMM never wrote, so + its `inputs` row is stale. `topk_ids >= 0` alone cannot tell it apart from a + real slot -- only expert_map can. This pins the contract the humming path + relies on (globalized ids are still routed through expert_map as a safety + net): with the map the stale slot is masked; without it, it corrupts the sum. + Regression guard against re-dropping expert_map as an optimization. + """ + set_random_seed(0) + device = "cuda" + num_tokens, top_k = 8, 8 + local = NUM_EXPERTS // 2 + expert_map = _local_expert_map(device) + + inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device) + topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device) + # Each row keeps at least one local slot (so the mapped output is finite) and + # at least one non-local slot whose stale row is poisoned with NaN. + half = top_k // 2 + topk_ids = torch.empty(num_tokens, top_k, dtype=torch.int32, device=device) + topk_ids[:, :half] = torch.randint( + 0, local, (num_tokens, half), dtype=torch.int32, device=device + ) + topk_ids[:, half:] = torch.randint( + local, NUM_EXPERTS, (num_tokens, top_k - half), dtype=torch.int32, device=device + ) + inputs[:, half:] = float("nan") + + atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4) + + mapped = moe_fused_mul_sum( + inputs=inputs, + topk_weights=topk_weights, + outputs=torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device), + topk_ids=topk_ids, + expert_map=expert_map, + ) + assert not mapped.isnan().any(), "expert_map failed to mask stale non-local slot" + ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map) + torch.testing.assert_close(mapped.float(), ref, atol=atol, rtol=rtol) + + # Same call without the map: the non-local slot is summed and the NaN leaks. + unmapped = moe_fused_mul_sum( + inputs=inputs, + topk_weights=topk_weights, + outputs=torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device), + topk_ids=topk_ids, + expert_map=None, + ) + assert unmapped.isnan().any(), "expert_map is not redundant: dropping it must leak" + + +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_num_valid_tokens_bounds_stale_padding(dtype: torch.dtype): + """Rows past num_valid_tokens are left untouched, even with stale ids/NaN. + + Under cudagraph decode the grid is the static padded token count and the + tail rows past num_recv hold stale (non -1) ids and stale NaN in `inputs` + from a prior replay -- the -1 mask alone cannot spot them. Bounding on + num_valid_tokens must skip the tail before any load so its NaN never reaches + the output; without the bound it is summed and a masked finalize can then + propagate the NaN. Regression guard for the decode accuracy fix. + """ + set_random_seed(0) + device = "cuda" + num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 512 + + inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device) + topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device) + # All ids valid (>= 0): the stale tail is indistinguishable by the -1 mask. + topk_ids = torch.randint( + 0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device + ) + inputs[num_recv:] = float("nan") # stale down_output behind the padding tail + num_valid = torch.tensor([num_recv], dtype=torch.int32, device=device) + + atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4) + + bounded = moe_fused_mul_sum( + inputs=inputs, + topk_weights=topk_weights, + outputs=torch.full( + (num_tokens, hidden_size), SENTINEL, dtype=dtype, device=device + ), + topk_ids=topk_ids, + num_valid_tokens=num_valid, + ) + assert torch.all(bounded[num_recv:] == SENTINEL), ( + "padding tail past num_recv written" + ) + assert not bounded[:num_recv].isnan().any(), "real rows leaked NaN" + ref = _reference( + inputs[:num_recv].nan_to_num(), + topk_weights[:num_recv], + topk_ids[:num_recv], + expert_map=None, + ) + torch.testing.assert_close(bounded[:num_recv].float(), ref, atol=atol, rtol=rtol) + + # Without the bound the stale tail is summed and leaks NaN into its rows. + unbounded = moe_fused_mul_sum( + inputs=inputs, + topk_weights=topk_weights, + outputs=torch.full( + (num_tokens, hidden_size), SENTINEL, dtype=dtype, device=device + ), + topk_ids=topk_ids, + ) + assert unbounded[num_recv:].isnan().any(), "num_valid_tokens bound is load-bearing" diff --git a/tests/kernels/moe/test_situ_mul_fp8_quant.py b/tests/kernels/moe/test_situ_mul_fp8_quant.py new file mode 100644 index 0000000000..3ad2833076 --- /dev/null +++ b/tests/kernels/moe/test_situ_mul_fp8_quant.py @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Correctness tests for the persistent cp.async-pipelined SITU + block-FP8 +quant kernel (``situ_and_mul_quant``). + +Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference +activation + block-FP8 quant, drive random per-row scale magnitudes, exercise +the masked (``valid_rows``) contiguous layout, and compare per valid row. + +The kernel computes SITU in fp32 and converts to fp8 with hardware rounding, +which torch cannot reproduce bit-exactly. It is therefore validated against the +*ideal* FP8 quantizer of the exact fp64 activation: the difference must stay +within the unavoidable FP8 block-quant error (within 1 fp8 code for ~all +elements, never off by more than 2). +""" + +import pytest +import torch + +from vllm.model_executor.layers.fused_moe.activation import situ_and_mul_quant +from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed + +fp8_dtype = current_platform.fp8_dtype() +DEVICE = current_platform.device_type +FP8_MAX = torch.finfo(fp8_dtype).max + +# The kernel uses fp8 e4m3 conversion, so it is CUDA-only and needs SM89+ +# (Ada/Hopper). There is no Triton fallback. +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda() or not current_platform.has_device_capability(89), + reason="situ_and_mul_quant persistent kernel requires CUDA SM89+ (fp8 e4m3).", +) + +# Kimi-K3 baked SITU params. Only (d=3072, beta=4.0, linear_beta=25.0, +# group_size=128, fp16/bf16) routes to the persistent cp.async-pipelined kernel; +# anything else falls back to the scalar group kernel. +SITU_BETA = 4.0 +SITU_LINEAR_BETA = 25.0 +SITU_D = 3072 +GROUP_SIZE = 128 + +# (num_tokens, valid_rows). valid_rows=None => all rows valid. 2048/4096 rows +# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8). +CASES = [ + (1, None), + (17, None), + (128, None), + (2048, None), + (4096, None), + (2048, 1500), # DeepEP v2 contiguous padding + (2048, 1), + (4096, 4095), + (512, 0), # whole tensor is padding +] + + +def token_random(num_tokens: int, twod: int, dtype: torch.dtype) -> torch.Tensor: + """Per-row random exponent so groups span many scale magnitudes.""" + base = torch.randn(num_tokens, twod, dtype=torch.float32, device=DEVICE) + exps = torch.randint(1, 13, (num_tokens, 1), device=DEVICE).float() + return (base * (2.0**exps)).to(dtype) + + +def ref_situ_block_fp8_quant( + inp: torch.Tensor, beta: float, linear_beta: float, group_size: int +) -> tuple[torch.Tensor, torch.Tensor]: + """Ideal FP8 block-quant of the exact fp64 SITU activation. + + The kernel rounds the activation to ``scalar_t`` before quantizing (it stores + ``(float)(scalar_t)situ(...)``), so the reference does the same before taking + the per-group absmax. + """ + d = inp.shape[-1] // 2 + gate, up = inp.double().chunk(2, dim=-1) + gate_out = beta * torch.tanh(gate / beta) * torch.sigmoid(gate) + up_out = linear_beta * torch.tanh(up / linear_beta) if linear_beta > 0 else up + act = (gate_out * up_out).to(inp.dtype).float() + + m = act.shape[0] + ng = d // group_size + a = act.view(m, ng, group_size) + scale = a.abs().amax(dim=2).clamp_min(1e-30) / FP8_MAX + q = (a / scale.unsqueeze(2)).clamp(-FP8_MAX, FP8_MAX).to(fp8_dtype) + return q.view(m, d), scale + + +def fp8_code_dist(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + """Signed-magnitude fp8 code distance (valid for e4m3fn and e4m3fnuz).""" + + def ordered(x: torch.Tensor) -> torch.Tensor: + x = x.view(torch.uint8).to(torch.int32) + return torch.where((x >> 7) == 1, -(x & 0x7F), x & 0x7F) + + return (ordered(a) - ordered(b)).abs() + + +@pytest.mark.parametrize("num_tokens,valid", CASES) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half]) +@torch.inference_mode() +def test_situ_and_mul_quant_pipelined( + num_tokens: int, valid: int | None, dtype: torch.dtype +) -> None: + set_random_seed(42) + d = SITU_D + inp = token_random(num_tokens, 2 * d, dtype) + + # Sentinels: padding rows must keep their bytes (out) and get scale 1.0. + out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype) + scale = torch.full( + (num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE + ) + out_sentinel = out.clone() + valid_rows = ( + None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE) + ) + + situ_and_mul_quant( + out, + scale, + inp, + beta=SITU_BETA, + linear_beta=SITU_LINEAR_BETA, + group_size=GROUP_SIZE, + valid_rows=valid_rows, + ) + + # Padding contract: skipped rows keep their bytes; their scale becomes 1.0. + if valid is not None and valid < num_tokens: + assert torch.equal(scale[valid:], torch.ones_like(scale[valid:])) + assert torch.equal( + out[valid:].view(torch.uint8), out_sentinel[valid:].view(torch.uint8) + ) + + rows = num_tokens if valid is None else valid + if rows == 0: + return + + ref_q, ref_s = ref_situ_block_fp8_quant( + inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE + ) + + assert torch.isfinite(scale[:rows]).all() + assert not torch.isnan(out[:rows].float()).any() + + # Scales: the kernel's fp32 SITU shifts the per-group absmax only + # marginally vs the fp64 reference. + torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5) + + # FP8 codes: must land within 1 code of the ideal quantizer for ~all + # elements, never off by more than 2. + dist = fp8_code_dist(out[:rows], ref_q) + assert dist.max().item() <= 2 + assert (dist <= 1).float().mean().item() >= 0.99 + + +@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)]) +@torch.inference_mode() +def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None: + """valid_rows is a token count; the kernel bounds rows at tokens*topk.""" + set_random_seed(7) + d = SITU_D + num_tokens = tokens * topk + 37 # padding tail past the valid rows + inp = token_random(num_tokens, 2 * d, torch.bfloat16) + + out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype) + scale = torch.full( + (num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE + ) + out_sentinel = out.clone() + valid_rows = torch.tensor(tokens, dtype=torch.int32, device=DEVICE) + + situ_and_mul_quant( + out, + scale, + inp, + beta=SITU_BETA, + linear_beta=SITU_LINEAR_BETA, + group_size=GROUP_SIZE, + valid_rows=valid_rows, + topk=topk, + ) + + rows = tokens * topk + assert torch.equal(scale[rows:], torch.ones_like(scale[rows:])) + assert torch.equal( + out[rows:].view(torch.uint8), out_sentinel[rows:].view(torch.uint8) + ) + + ref_q, ref_s = ref_situ_block_fp8_quant( + inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE + ) + torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5) + assert fp8_code_dist(out[:rows], ref_q).max().item() <= 2 + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half]) +@torch.inference_mode() +def test_situ_and_mul_quant_pipelined_deterministic(dtype: torch.dtype) -> None: + """Two runs on the same input must be byte-identical.""" + set_random_seed(0) + d = SITU_D + num_tokens = 2048 + inp = token_random(num_tokens, 2 * d, dtype) + + def run() -> tuple[torch.Tensor, torch.Tensor]: + out = torch.empty(num_tokens, d, dtype=fp8_dtype, device=DEVICE) + scale = torch.empty( + num_tokens, d // GROUP_SIZE, dtype=torch.float32, device=DEVICE + ) + situ_and_mul_quant( + out, + scale, + inp, + beta=SITU_BETA, + linear_beta=SITU_LINEAR_BETA, + group_size=GROUP_SIZE, + ) + return out, scale + + out1, s1 = run() + out2, s2 = run() + assert torch.equal(out1.view(torch.uint8), out2.view(torch.uint8)) + assert torch.equal(s1, s2) diff --git a/tests/quantization/test_humming_mxfp4_block_fp8.py b/tests/quantization/test_humming_mxfp4_block_fp8.py new file mode 100644 index 0000000000..dbec989df3 --- /dev/null +++ b/tests/quantization/test_humming_mxfp4_block_fp8.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for Humming MXFP4-weight GEMM paired with a block-FP8 activation. + +The Humming kernel already implements MXFP4 weight (group-32 e8m0) with a +grouped FP8 activation via WGMMA software dequant -- see the grouped-fp8 +`test_mxfp4.py` cases in the humming repo, which run on SM90 (H200). Enabling +it in vLLM comes down to two things, both exercised here: + + 1. A grouped FP8 activation must classify to a float32-scaled key + (kFp8Dynamic128Sym), not a uint8 microscale key. MX microscale + activations (group size 32) keep their e8m0 (uint8) scale. + 2. (kMxfp4Static, kFp8Dynamic128Sym) must be an allowed MoE + (weight, activation) pair. +""" + +import pytest + +from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + HummingExpertsBase, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8Dynamic128Sym, + kFp8DynamicTokenSym, + kMxfp4Static, + kMxfp8Dynamic, +) +from vllm.utils.import_utils import has_humming + +# make_humming_moe_quant_config stores per-sublayer humming LayerConfigs but the +# config-plumbing asserts below never read them, so a placeholder satisfies the +# required arg without pulling in the humming package. +_PLACEHOLDER_HUMMING_CONFIGS = {"w13": None, "w2": None} + + +def test_mxfp4_weight_with_block_fp8_activation_is_supported(): + """MXFP4 weight + block-FP8 (group-128) activation is an allowed pair.""" + assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8Dynamic128Sym) + + +def test_mxfp4_weight_with_per_token_fp8_activation_still_supported(): + """The pre-existing per-token FP8 pairing must keep working.""" + assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8DynamicTokenSym) + + +@pytest.mark.skipif(not has_humming(), reason="humming is not installed") +@pytest.mark.parametrize( + "group_size,expected_key", + [ + # per-token FP8 -> float32 per-token scale + (0, kFp8DynamicTokenSym), + # block FP8 -> float32 group-128 scale (the newly-fixed case) + (128, kFp8Dynamic128Sym), + # MXFP8 microscale -> e8m0 (uint8) group-32 scale (must be preserved) + (32, kMxfp8Dynamic), + ], +) +def test_humming_fp8_input_schema_to_quant_key(group_size, expected_key): + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + _humming_input_schema_to_quant_key, + ) + from vllm.utils.humming import HummingInputSchema + from vllm.utils.humming import dtypes as humming_dtypes + + schema = HummingInputSchema( + a_dtype=humming_dtypes.float8e4m3, + input_scale_group_size=group_size, + ) + assert _humming_input_schema_to_quant_key(schema) == expected_key + + +@pytest.mark.skipif(not has_humming(), reason="humming is not installed") +def test_humming_bf16_input_schema_is_unquantized(): + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + _humming_input_schema_to_quant_key, + ) + from vllm.utils.humming import HummingInputSchema + + # No a_dtype -> unquantized (bf16/fp16) inputs -> None. + assert _humming_input_schema_to_quant_key(HummingInputSchema()) is None + + +def test_block_fp8_activation_quant_config_is_block_quantized(): + """A block-FP8 activation group shape yields a block-quantized MoE config + (FP8 dtype, [1, 128] block, not per-act-token). This is what makes the + DeepEP prepare/finalize step quantize activations to block FP8 *before* the + all-to-all dispatch instead of deferring to Humming -- and what makes + HummingExpertsBase.expects_unquantized_inputs return False.""" + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + make_humming_moe_quant_config, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape + from vllm.platforms import current_platform + + qc = make_humming_moe_quant_config( + quant_dtype=current_platform.fp8_dtype(), + weight_dtype="float4e2m1", + activation_group_shape=GroupShape(row=1, col=128), + humming_configs=_PLACEHOLDER_HUMMING_CONFIGS, + ) + assert qc.is_block_quantized + assert qc.block_shape == [1, 128] + assert not qc.per_act_token_quant + assert qc.quant_dtype == current_platform.fp8_dtype() + + +def test_default_activation_quant_config_defers_to_humming(): + """Without a block activation shape the config stays per-token (the deferred + path): Humming quantizes internally, preserving pre-existing behavior for + every non-block-FP8 scheme.""" + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + make_humming_moe_quant_config, + ) + from vllm.platforms import current_platform + + qc = make_humming_moe_quant_config( + quant_dtype=current_platform.fp8_dtype(), + weight_dtype="float4e2m1", + humming_configs=_PLACEHOLDER_HUMMING_CONFIGS, + ) + assert not qc.is_block_quantized + assert qc.per_act_token_quant diff --git a/vllm/model_executor/layers/fused_moe/activation.py b/vllm/model_executor/layers/fused_moe/activation.py index 6585bbe4f4..e9212cacf7 100644 --- a/vllm/model_executor/layers/fused_moe/activation.py +++ b/vllm/model_executor/layers/fused_moe/activation.py @@ -201,6 +201,7 @@ def apply_moe_activation( activation_config: ApplyMoEActivationConfig | None = None, topk_ids: torch.Tensor | None = None, expert_map: torch.Tensor | None = None, + valid_rows: torch.Tensor | None = None, ) -> torch.Tensor: """Apply MoE activation function. @@ -256,6 +257,7 @@ def apply_moe_activation( -1.0 if config.activation_situ_linear_beta is None else config.activation_situ_linear_beta, + valid_rows, ) elif activation == MoEActivation.SWIGLUOAI: torch.ops._C.swigluoai_and_mul(output, input) @@ -286,3 +288,45 @@ def apply_moe_activation( raise ValueError(f"Unsupported FusedMoe activation: {activation}") return output + + +def situ_and_mul_quant( + output: torch.Tensor, + scale: torch.Tensor, + input: torch.Tensor, + *, + beta: float, + linear_beta: float | None, + group_size: int = 0, + valid_rows: torch.Tensor | None = None, + topk: int = 1, +) -> None: + """Fused Kimi SITU activation + dynamic FP8 quantization. + + Writes the quantized fp8 down-projection input into ``output`` [M, d] and its + float32 scale into ``scale`` (dequant = q * scale), fusing the SITU + activation with the FP8 quant that Humming's w2 GEMM would otherwise perform + in a separate pass -- this avoids materializing (and rounding through) the + bf16 activation_output buffer. + + ``group_size`` selects the quant granularity: ``0`` gives per-token scales + ``scale`` [M, 1]; ``128`` gives k-major block-FP8 group scales ``scale`` + [M, d // 128], matching humming ``quant_input(group_size=128, float32)``. + + ``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching + ``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the + DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the + kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply + is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept + in this module so that every ``torch.ops._C.situ*`` call lives in one file. + """ + torch.ops._C.situ_and_mul_quant( + output, + scale, + input, + beta, + -1.0 if linear_beta is None else linear_beta, + group_size, + valid_rows, + topk, + ) diff --git a/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py b/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py index b25d3e93ae..70158dafa1 100644 --- a/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py +++ b/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py @@ -525,7 +525,9 @@ def deepgemm_moe_permute( expert_num_tokens = None if expert_tokens_meta is not None: expert_num_tokens = expert_tokens_meta.expert_num_tokens - else: + if expert_num_tokens is None: + # A cudagraph-decode carrier may hold only psum_recv_per_rank and leave + # expert_num_tokens None; count locally as in the meta-absent case. expert_num_tokens = count_expert_num_tokens( topk_ids, local_num_experts, expert_map ) diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py index 40d17b14dc..bad632071a 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py @@ -668,6 +668,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular): assert hidden_states.dim() == 3 assert expert_tokens_meta is not None expert_num_tokens = expert_tokens_meta.expert_num_tokens + assert expert_num_tokens is not None num_local_experts = w1.size(0) assert num_local_experts == w1.size(0), f"{num_local_experts} == {w1.size(0)}" diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py index 69843c228a..3a85549133 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py @@ -119,6 +119,37 @@ def get_humming_moe_gemm_type() -> str: return gemm_type +def _fixup_moe_tuning_config(tuning_config: list, max_k_block: int = 128) -> None: + """Fix up each MoE tile in place: cap the K-block and widen warp-N. + + - block_shape[2] (K-block) > ``max_k_block``: the driver rejects the TMA + descriptor at launch (CUDA_ERROR_MISALIGNED_ADDRESS). Cap at 128, which + Humming already uses for larger M. + - warp_shape[1] (warp-N) < 32: block-FP8 (group-128) activations route the + w13 (gate/up) GEMM to a tuning table that pins warp-N to 16, under-filling + the Hopper WGMMA N dimension and corrupting the GEMM output (gsm8k + 0.94 -> 0.89). Widen to 32 whenever block_n % 32 == 0 -- on every tile, + not just the K-capped ones. w2 (down) already uses warp_n=32. + """ + logger.info_once("Attempting to override humming GEMM config") + for entry in tuning_config: + config = entry[2] + block_shape = config.get("block_shape") + if not (block_shape and len(block_shape) == 3): + continue + block_m, block_n, block_k = block_shape + + logger.info_once(f"Overriding humming GEMM config. Previous config\n: {config}") + if block_k > max_k_block: + config["block_shape"] = [block_m, block_n, max_k_block] + + warp_shape = config.get("warp_shape") + if warp_shape and warp_shape[1] < 32 and block_n % 32 == 0: + config["warp_shape"] = [warp_shape[0], 32, warp_shape[2]] + + logger.info_once(f"Overridden humming GEMM config. Current config\n: {config}") + + class HummingExpertsBase(mk.FusedMoEExpertsModular): def __init__( self, @@ -170,6 +201,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): gemm_type=self.humming_gemm_type(), ) self.compute_config_str = json.dumps(self.compute_config) + # Fix up the heuristic-chosen tiles before freezing to JSON -- see + # _fixup_moe_tuning_config. + _fixup_moe_tuning_config(self.w13_tuning_config) + _fixup_moe_tuning_config(self.w2_tuning_config) self.w13_tuning_config_str = json.dumps(self.w13_tuning_config) self.w2_tuning_config_str = json.dumps(self.w2_tuning_config) @@ -178,12 +213,17 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): sublayer_name: str, inputs: torch.Tensor, quanted_input: torch.Tensor | None, + input_scale: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor | None]: from vllm.utils.humming import may_quant_input + # input_scale is set for block-FP8 (group-128) activations that were + # quantized before the EP dispatch: may_quant_input then skips the + # redundant w13 quantization and forwards the pre-computed scale. return may_quant_input( self.humming_configs[sublayer_name], inputs=inputs, + input_scale=input_scale, quanted_input=quanted_input, ) @@ -263,6 +303,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): (kMxfp4Static, kMxfp4Dynamic), (kMxfp4Static, kMxfp8Dynamic), (kMxfp4Static, kFp8DynamicTokenSym), + # MXFP4 weight (group-32 e8m0) with block-FP8 activation + # (group-128 float32). Runs via WGMMA software dequant, so it + # works on Hopper (SM90/H200) as well as Blackwell. + (kMxfp4Static, kFp8Dynamic128Sym), (kNvfp4Static, None), (kNvfp4Static, kFp8DynamicTokenSym), (kMxfp8Static, None), @@ -299,21 +343,43 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): and _is_supported_wna16_weight_key(weight_key) ) + def _prequantizes_dispatch_activation(self) -> bool: + """ + Whether the prepare/finalize step should quantize activations before + the (EP all-to-all) dispatch instead of leaving it to Humming. + + This is enabled only for block-FP8 (group-128) activations: quantizing + to FP8 before dispatch sends FP8 rather than BF16 over the interconnect, + and Humming then consumes the pre-quantized FP8 + scale directly (see + the apply() methods, which forward the dispatch scale into + HummingMethod.may_quant_input, and may_quant_input itself, which is a + no-op when an input scale is already supplied). The scale layout + produced by vLLM's block-FP8 quantization ([M, K // 128] float32, + row-major) matches what the Humming WGMMA grouped GEMM expects. + """ + quant_config = self.quant_config + return ( + quant_config.is_block_quantized + and quant_config.quant_dtype == current_platform.fp8_dtype() + ) + @property def expects_unquantized_inputs(self) -> bool: """ - Humming kernels handle input quantization internally in apply(). + Whether the prepare/finalize step should defer input quantization to + the experts (by setting defer_input_quant=True and passing unquantized + inputs). - This property tells the prepare/finalize step to skip input - quantization (by setting defer_input_quant=True) and pass - unquantized inputs to the experts. This prevents double - quantization: once in prepare and once in Humming's apply(). + Humming normally quantizes inputs internally via + HummingMethod.may_quant_input() in apply(), so we defer quantization + (return True) to avoid quantizing twice -- once in prepare and once in + Humming's apply(). - Returns: - True to indicate that this expert expects unquantized inputs - and will handle quantization internally. + The exception is block-FP8 (group-128) activations, which are quantized + before the dispatch to save interconnect bandwidth (see + _prequantizes_dispatch_activation): for those we must NOT defer. """ - return True + return not self._prequantizes_dispatch_activation() @staticmethod def _supports_current_device() -> bool: @@ -585,6 +651,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): activation: MoEActivation, output: torch.Tensor, input: torch.Tensor, + valid_rows: torch.Tensor | None = None, ) -> None: activation_config = self.activation_config if ( @@ -601,8 +668,84 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): activation=activation, input=input, output=output, + valid_rows=valid_rows, ) + def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool: + """Whether the SITU activation + w2 quant can be fused into one kernel. + + Fused only for k-major block-FP8 group-128 e4m3 with float32 scales -- + the sole layout situ_and_mul_quant supports. A float32 as_dtype rules out + MXMMA (which uses e8m0 m-major scales), so a group-128 scale is + guaranteed k-major. Everything else (m-major MXFP8, other group sizes, + 16-bit passthrough, non-SITU) falls back to the separate + situ_and_mul + quantize_input path. + """ + if activation != MoEActivation.SITU: + return False + w2cfg = self.humming_configs["w2"] + # Report the specific blocker once so a silent fallback to the unfused + # situ_and_mul + quant pair is never a mystery in a trace. + reason: str | None = None + if not (w2cfg.a_dtype.num_bits == 8 and str(w2cfg.a_dtype) == "float8e4m3"): + reason = f"w2 a_dtype is {w2cfg.a_dtype} (need float8e4m3)" + elif w2cfg.input_scale_group_size != 128: + reason = ( + f"w2 input_scale_group_size is {w2cfg.input_scale_group_size} " + "(need 128)" + ) + elif not (w2cfg.as_dtype is not None and str(w2cfg.as_dtype) == "float32"): + reason = f"w2 as_dtype is {w2cfg.as_dtype} (need float32, k-major)" + if reason is not None: + logger.warning_once( + "Humming fused SITU+FP8 quant disabled, using unfused " + "situ_and_mul + quant: %s", + reason, + ) + return False + return True + + def fused_situ_quant( + self, + gate_up_output: torch.Tensor, + quanted_down_input: torch.Tensor, + valid_rows: torch.Tensor | None, + topk: int = 1, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Run the fused SITU activation + FP8 quant for the w2 input. + + Returns ``(quanted_down_input, input_scale)`` in the same layout the + unfused ``apply_activation`` + ``quantize_input("w2")`` pair produces: an + fp8 [M, d] tensor plus a k-major block-FP8 float32 [M, d // 128] group + scale. Only reached when ``fused_situ_quant_enabled`` (group_size 128). + """ + from vllm.model_executor.layers.fused_moe.activation import ( + situ_and_mul_quant, + ) + + cfg = self.activation_config + assert cfg.activation_situ_beta is not None, ( + "SITU requires activation_situ_beta from FusedMoEConfig" + ) + group_size = self.humming_configs["w2"].input_scale_group_size + m, d = quanted_down_input.size(0), quanted_down_input.size(1) + input_scale = torch.empty( + (m, d // group_size), + dtype=torch.float32, + device=quanted_down_input.device, + ) + situ_and_mul_quant( + quanted_down_input, + input_scale, + gate_up_output, + beta=cfg.activation_situ_beta, + linear_beta=cfg.activation_situ_linear_beta, + group_size=group_size, + valid_rows=valid_rows, + topk=topk, + ) + return quanted_down_input, input_scale + class HummingIndexedExperts(HummingExpertsBase): def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: @@ -684,10 +827,12 @@ class HummingIndexedExperts(HummingExpertsBase): """ Standard apply implementation for Humming indexed experts. - Humming performs activation quantization internally and consumes the - weights supplied by the modular kernel interface. - Results are written to the output buffer supplied by the modular - kernel, which may alias workspace13. + Note: Humming kernels handle weights internally through the layer + object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual + path (Humming quantizes the w13 input itself); for block-FP8 activations + it carries the scale computed before dispatch, which is forwarded to + may_quant_input so Humming skips the redundant w13 quantization. The + output is written into workspace13 via the buffer management. """ assert not apply_router_weight_on_input @@ -710,6 +855,7 @@ class HummingIndexedExperts(HummingExpertsBase): inputs, input_scale = self.quantize_input( "w13", inputs=hidden_states, + input_scale=a1q_scale, quanted_input=buffers.get("quanted_gate_up_input", None), ) @@ -722,17 +868,45 @@ class HummingIndexedExperts(HummingExpertsBase): **moe_kwargs1, ) - self.apply_activation( - activation=activation, - input=buffers["gate_up_output"], - output=buffers["activation_output"], - ) + # psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view. + # situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds + # on tokens, so both read this pointer directly -- no host cast/multiply. + valid_tokens = None + topk = topk_ids.size(1) + if ( + expert_tokens_meta is not None + and expert_tokens_meta.psum_recv_per_rank is not None + ): + valid_tokens = expert_tokens_meta.psum_recv_per_rank[-1:] - inputs, input_scale = self.quantize_input( - "w2", - inputs=buffers["activation_output"], - quanted_input=buffers.get("quanted_down_input", None), - ) + if self.fused_situ_quant_enabled(activation): + # Fused SITU + FP8 quant (per-token or block-FP8 group-128) straight + # into the w2 input, skipping the bf16 activation_output round-trip. + inputs, input_scale = self.fused_situ_quant( + gate_up_output=buffers["gate_up_output"], + quanted_down_input=buffers["quanted_down_input"], + valid_rows=valid_tokens, + topk=topk, + ) + else: + # Fallback situ_and_mul takes int64 row counts (tokens*topk). + valid_rows = ( + valid_tokens.to(torch.int64) * topk + if valid_tokens is not None + else None + ) + self.apply_activation( + activation=activation, + input=buffers["gate_up_output"], + output=buffers["activation_output"], + valid_rows=valid_rows, + ) + + inputs, input_scale = self.quantize_input( + "w2", + inputs=buffers["activation_output"], + quanted_input=buffers.get("quanted_down_input", None), + ) self.humming_forward( "w2", @@ -743,12 +917,16 @@ class HummingIndexedExperts(HummingExpertsBase): **moe_kwargs2, ) + # expert_map masks any non-local id; num_valid_tokens bounds the + # persistent kernel to the real token rows [0, num_recv) so the padding + # tail is never iterated (CUDA-graph-safe device scalar). moe_fused_mul_sum( inputs=buffers["down_output"].view(*topk_ids.shape, -1), topk_weights=topk_weights, topk_ids=topk_ids, expert_map=expert_map, outputs=output, + num_valid_tokens=valid_tokens, ) @@ -787,10 +965,13 @@ class HummingGroupedExperts(HummingExpertsBase): """ Standard apply implementation for Humming grouped experts. - Humming performs activation quantization internally and consumes the - weights supplied by the modular kernel interface. - Results are written to the output buffer supplied by the modular - kernel, which may alias workspace13. + Note: Humming kernels handle weights internally through the layer + object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual + path (Humming quantizes the w13 input itself); for block-FP8 activations + it carries the scale computed before dispatch. It is permuted alongside + the tokens by moe_permute and forwarded to may_quant_input so Humming + skips the redundant w13 quantization. The output is written into + workspace13 via the buffer management. """ assert not apply_router_weight_on_input @@ -805,9 +986,9 @@ class HummingGroupedExperts(HummingExpertsBase): ) buffers["output"] = output - hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute( + hidden_states, a1q_scale, expert_first_token_offset, inv_perm, _ = moe_permute( hidden_states=hidden_states, - a1q_scale=None, + a1q_scale=a1q_scale, topk_ids=topk_ids, n_expert=global_num_experts, n_local_expert=self.num_experts, @@ -818,6 +999,7 @@ class HummingGroupedExperts(HummingExpertsBase): inputs, input_scale = self.quantize_input( "w13", inputs=hidden_states, + input_scale=a1q_scale, quanted_input=buffers.get("quanted_gate_up_input", None), ) @@ -901,15 +1083,21 @@ class BatchedHummingGroupedExperts(HummingExpertsBase): """ Standard apply implementation for Humming batched grouped experts. - Humming performs activation quantization internally and consumes the - weights supplied by the modular kernel interface. - Results are written to the output buffer supplied by the modular - kernel, which may alias workspace13. + Note: Humming kernels handle weights internally through the layer + object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual + path (Humming quantizes the w13 input itself); for block-FP8 activations + it carries the scale computed before dispatch, which is forwarded to + may_quant_input so Humming skips the redundant w13 quantization. The + output is written into workspace13 via the buffer management. """ assert not apply_router_weight_on_input assert expert_tokens_meta is not None hidden_states = hidden_states.view(-1, hidden_states.size(-1)) + # Keep the (batched) block-FP8 scale row-aligned with the flattened + # [num_experts * max_tokens, K] hidden states above. + if a1q_scale is not None and a1q_scale.dim() == 3: + a1q_scale = a1q_scale.view(-1, a1q_scale.size(-1)) valid_shape_m = self.estimate_local_valid_shape_m(topk_ids) expert_num_tokens = expert_tokens_meta.expert_num_tokens @@ -925,6 +1113,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase): inputs, input_scale = self.quantize_input( "w13", inputs=hidden_states, + input_scale=a1q_scale, quanted_input=buffers.get("quanted_gate_up_input", None), ) diff --git a/vllm/model_executor/layers/fused_moe/modular_kernel.py b/vllm/model_executor/layers/fused_moe/modular_kernel.py index c159f059d6..90736c09a1 100644 --- a/vllm/model_executor/layers/fused_moe/modular_kernel.py +++ b/vllm/model_executor/layers/fused_moe/modular_kernel.py @@ -100,8 +100,9 @@ class ExpertTokensMetadata: Metadata regarding expert-token routing. """ - expert_num_tokens: torch.Tensor + expert_num_tokens: torch.Tensor | None expert_num_tokens_cpu: torch.Tensor | None + psum_recv_per_rank: torch.Tensor | None = None @staticmethod def make_from_list( @@ -901,6 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts): *, topk_ids: torch.Tensor | None = None, expert_map: torch.Tensor | None = None, + valid_rows: torch.Tensor | None = None, ) -> None: apply_moe_activation( activation, @@ -909,6 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts): activation_config=self.activation_config, topk_ids=topk_ids, expert_map=expert_map, + valid_rows=valid_rows, ) @abstractmethod diff --git a/vllm/model_executor/layers/fused_moe/moe_fused_mul_sum.py b/vllm/model_executor/layers/fused_moe/moe_fused_mul_sum.py index 768f41db85..f9854d1773 100644 --- a/vllm/model_executor/layers/fused_moe/moe_fused_mul_sum.py +++ b/vllm/model_executor/layers/fused_moe/moe_fused_mul_sum.py @@ -3,7 +3,6 @@ import torch from torch._subclasses.fake_tensor import FakeTensor -from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -14,122 +13,75 @@ def moe_fused_mul_sum_kernel( outputs_ptr, top_ids_ptr, expert_map_ptr, - num_tokens, + num_valid_tokens_ptr, stride_m, + has_topk_ids: tl.constexpr, has_expert_map: tl.constexpr, + has_num_valid: tl.constexpr, top_k: tl.constexpr, - size: tl.constexpr, - BLOCK_M: tl.constexpr, + hidden_size: tl.constexpr, BLOCK_K: tl.constexpr, ): - pid_k = tl.program_id(0) - pid_m = tl.program_id(1) + # One CTA owns one token's output row. Resolve which slots to sum once per + # row (the decision is invariant over hidden tiles); the `if take` branch is + # block-uniform, so padding slots are skipped rather than masked-and-loaded. + pid_m = tl.program_id(0) + # Bound to the real recv rows [0, num_recv). Under cudagraph decode the grid + # is the static padded token count, and padding rows past num_recv carry + # stale ids and stale NaN in `inputs`; returning before any load leaves their + # output rows untouched instead of summing that garbage. The constexpr guard + # short-circuits, so num_valid_tokens_ptr is only read when it is provided. + if has_num_valid and pid_m >= tl.load(num_valid_tokens_ptr).to(tl.int32): + return + w_row = topk_weights_ptr + pid_m * top_k - offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) - offs_k = pid_k * BLOCK_K + tl.arange(0, BLOCK_K) + takes = () + weights = () + if has_topk_ids: + any_valid = 0 + for n in tl.static_range(top_k): + id_val = tl.load(top_ids_ptr + pid_m * top_k + n) + valid = id_val >= 0 + any_valid += valid.to(tl.int32) + take = valid + if has_expert_map: + local_id = tl.load(expert_map_ptr + tl.where(valid, id_val, 0)) + take = valid & (local_id >= 0) + takes += (take,) # type: ignore[assignment] + weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment] + # All-padding rows early-return untouched (decode contract past num_recv). + if any_valid == 0: + return + else: + for n in tl.static_range(top_k): + takes += (True,) # type: ignore[assignment] + weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment] - m_mask = offs_m < num_tokens - k_mask = offs_k < size - mask = m_mask[:, None] & k_mask[None, :] + n_tiles: tl.constexpr = (hidden_size + BLOCK_K - 1) // BLOCK_K + a_row = inputs_ptr + pid_m * stride_m + out_row = outputs_ptr + pid_m * hidden_size - a_base = inputs_ptr + (offs_m * stride_m)[:, None] + offs_k[None, :] - b_base = topk_weights_ptr + offs_m * top_k - - acc = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32) - - for n in tl.static_range(top_k): - b_val = tl.load(b_base + n, mask=m_mask, other=0.0).to(tl.float32) - if has_expert_map: - id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0) - expert_mask = tl.load(expert_map_ptr + id_val) >= 0 - a_vec = tl.load( - a_base + n * size, - mask=mask & expert_mask[:, None], - other=0.0, - ).to(tl.float32) - else: - a_vec = tl.load( - a_base + n * size, - mask=mask, - other=0.0, - ).to(tl.float32) - acc += a_vec * b_val[:, None] - - out_ptrs = outputs_ptr + (offs_m * size)[:, None] + offs_k[None, :] - tl.store( - out_ptrs, - acc.to(outputs_ptr.dtype.element_ty), - mask=mask, - ) + for t in tl.range(0, n_tiles): + offs_k = t * BLOCK_K + tl.arange(0, BLOCK_K) + kmask = offs_k < hidden_size + acc = tl.zeros((BLOCK_K,), dtype=tl.float32) + for n in tl.static_range(top_k): + if takes[n]: + a = tl.load(a_row + n * hidden_size + offs_k, mask=kmask, other=0.0) + acc += a.to(tl.float32) * weights[n] + tl.store(out_row + offs_k, acc.to(outputs_ptr.dtype.element_ty), mask=kmask) def _heuristic_config( - num_tokens: int, - top_k: int, - size: int, + hidden_size: int, element_size: int, ): is_fp32 = element_size > 2 - is_sm90_plus = current_platform.has_device_capability(90) - is_sm80_before = not current_platform.has_device_capability(80) - - if current_platform.has_device_capability(90): - # SM90/SM100+: prefer small tiles + many CTAs. - if is_fp32: - BLOCK_M = 1 if num_tokens <= 4 else 2 - else: - if num_tokens <= 4: - BLOCK_M = 1 - elif num_tokens <= 128: - BLOCK_M = 2 - else: - BLOCK_M = 4 - elif is_fp32: - if num_tokens <= 4: - BLOCK_M = 1 - elif num_tokens <= 32: - BLOCK_M = 2 - elif num_tokens <= 128: - BLOCK_M = 4 - else: - BLOCK_M = 4 - else: - if num_tokens <= 4: - BLOCK_M = 1 - elif num_tokens <= 32: - BLOCK_M = 2 - elif num_tokens <= 128: - BLOCK_M = 4 - elif num_tokens <= 1024: - BLOCK_M = 16 - else: - BLOCK_M = 8 - - if is_fp32: - max_block_k = 256 - elif is_sm80_before or is_sm90_plus: - max_block_k = 512 - else: - max_block_k = 1024 - BLOCK_K = min(triton.next_power_of_2(size), max_block_k) - BLOCK_K = max(BLOCK_K, 256) - - total = BLOCK_M * BLOCK_K - if is_fp32: - num_warps = max(8, min(16, total // 64)) - else: - num_warps = max(4, min(16, total // 256)) - - if is_sm80_before: - num_warps = min(num_warps, 8) - num_stages = 2 - elif is_sm90_plus: - num_warps = min(num_warps, 8) - num_stages = 4 if total <= 2048 else 2 - else: - num_stages = 4 if total <= 2048 else 2 - - return BLOCK_M, BLOCK_K, num_warps, num_stages + max_block_k = 256 if is_fp32 else 512 + BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k)) + num_warps = 4 if is_fp32 else 2 + num_stages = 3 + return BLOCK_K, num_warps, num_stages def moe_fused_mul_sum( @@ -138,6 +90,7 @@ def moe_fused_mul_sum( outputs: torch.Tensor | None = None, topk_ids: torch.Tensor | None = None, expert_map: torch.Tensor | None = None, + num_valid_tokens: torch.Tensor | None = None, ) -> torch.Tensor: """ Fused kernel for MoE (Mixture of Experts) to perform weighted summation @@ -150,10 +103,21 @@ def moe_fused_mul_sum( Shape: (num_tokens, top_k). outputs: Optional pre-allocated output tensor. Shape: (num_tokens, hidden_size). - topk_ids: Optional indices of the top-k experts. Used when - `expert_map` is provided. Shape: (num_tokens, top_k). + topk_ids: Optional indices of the top-k experts. Shape: + (num_tokens, top_k). A value of -1 marks a slot the expert GEMM + skipped; those slots are excluded from the sum. When provided, rows + with all top ids < 0 (worst-case padding) are skipped and their + output rows left untouched. Required when `expert_map` is provided. expert_map: Optional mapping for Expert Parallelism. A value < 0 - indicates an invalid token/expert pair that will be skipped. + indicates an invalid token/expert pair that will be skipped. Only + needed when `topk_ids` may contain non-local expert ids; if every + non-(-1) id is already a local expert, leave it None to skip the + redundant per-slot lookup. + num_valid_tokens: Optional device scalar (1-element tensor) holding the + number of real token rows (num_recv for a decode dispatch). When + provided, rows past it are left untouched, so the static cudagraph + grid never sums stale padding rows. Pass the token count, not + token*top_k. Returns: The fused weighted sum of expert outputs. @@ -166,34 +130,40 @@ def moe_fused_mul_sum( assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16) assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16) - num_tokens, top_k, size = inputs.shape - output_shape = (num_tokens, size) + num_tokens, top_k, hidden_size = inputs.shape + output_shape = (num_tokens, hidden_size) if outputs is None: outputs = torch.empty(output_shape, dtype=inputs.dtype, device=inputs.device) assert outputs.shape == output_shape assert topk_weights.shape == (num_tokens, top_k) + assert expert_map is None or topk_ids is not None, ( + "topk_ids is required to interpret expert_map" + ) + if topk_ids is not None: + assert topk_ids.shape == (num_tokens, top_k) + assert topk_ids.is_contiguous() + assert topk_ids.dtype in (torch.int32, torch.int64) if not isinstance(inputs, FakeTensor): - BLOCK_M, BLOCK_K, num_warps, num_stages = _heuristic_config( - num_tokens, - top_k, - size, + BLOCK_K, num_warps, num_stages = _heuristic_config( + hidden_size, inputs.element_size(), ) - grid = (triton.cdiv(size, BLOCK_K), triton.cdiv(num_tokens, BLOCK_M)) + grid = (num_tokens,) moe_fused_mul_sum_kernel[grid]( inputs, topk_weights, outputs, topk_ids, expert_map, - num_tokens, - top_k * size, + num_valid_tokens, + top_k * hidden_size, + topk_ids is not None, expert_map is not None, + num_valid_tokens is not None, top_k, - size, - BLOCK_M, + hidden_size, BLOCK_K, num_warps=num_warps, num_stages=num_stages, diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/deepep_v2.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/deepep_v2.py index 916cebd74c..139bf22fa8 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/deepep_v2.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/deepep_v2.py @@ -32,7 +32,7 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): - Worst-case tensor allocation; padding rows zeroed via handle.psum_num_recv_tokens_per_scaleup_rank - Fully cudagraph-capturable - - Expert kernel sorts internally (expert_tokens_meta=None) + - Expert kernel sorts internally (expert_tokens_meta carries no counts) **Prefill mode (use_cudagraph=False):** - do_expand=True, do_cpu_sync=True @@ -266,6 +266,18 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): if recv_topk_weights is not None and recv_topk_weights.ndim == 1: recv_topk_weights = recv_topk_weights.unsqueeze(1) + if self.use_cudagraph: + # Carry the per-rank prefix sum so SiTU can skip padding rows. + # expert_num_tokens stays None: count-based consumers (DeepGEMM, + # Triton) must treat a None field as "no counts" and derive their + # own, exactly as in the meta-absent decode case. + if expert_tokens_meta is None: + expert_tokens_meta = mk.ExpertTokensMetadata( + expert_num_tokens=None, + expert_num_tokens_cpu=None, + ) + expert_tokens_meta.psum_recv_per_rank = psum_recv_per_rank + if not quant_config.is_block_quantized and not defer_input_quant: expert_x_scale = None if expert_x.numel() != 0: diff --git a/vllm/model_executor/layers/quantization/utils/humming_utils.py b/vllm/model_executor/layers/quantization/utils/humming_utils.py index 1303a92d91..2a3c1bd4f2 100644 --- a/vllm/model_executor/layers/quantization/utils/humming_utils.py +++ b/vllm/model_executor/layers/quantization/utils/humming_utils.py @@ -307,7 +307,18 @@ def _humming_input_schema_to_quant_key( gs = schema.input_scale_group_size group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN - scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32 + # Pick the scale dtype the Humming kernel actually consumes. An explicit + # input_scale_dtype always wins. Otherwise infer from the grouping: MX + # microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8) + # scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a + # float32 scale. Getting this right lets a grouped FP8 activation match + # kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key. + if schema.input_scale_dtype is not None: + scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype] + elif gs == 32: + scale_dtype = MXFP_SCALE_DTYPE + else: + scale_dtype = torch.float32 scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape) @@ -553,6 +564,7 @@ def make_humming_moe_quant_config( quant_dtype: torch.dtype | str | None, weight_dtype: torch.dtype | str | None, weight_group_shape: GroupShape | None = None, + activation_group_shape: GroupShape | None = None, w1_scale: torch.Tensor | None = None, w2_scale: torch.Tensor | None = None, w1_zp: torch.Tensor | None = None, @@ -569,7 +581,14 @@ def make_humming_moe_quant_config( assert humming_configs is not None if quant_dtype is None: a_quant_desc = FusedMoEQuantDesc(dtype=None) + elif activation_group_shape is not None: + # Pre-dispatch quantization. + a_quant_desc = FusedMoEQuantDesc( + dtype=quant_dtype, shape=activation_group_shape + ) else: + # Deferred path: Humming quantizes the activation internally, so the + # descriptor only needs a non-None dtype to mark it as quantized. shape = GroupShape(row=1, col=-1) a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape) @@ -621,6 +640,19 @@ def get_humming_moe_quant_config( else: q_dtype = str(input_schema.a_dtype) + # Block-FP8 (group-128) activations are quantized *before* the EP all-to-all + # dispatch (so FP8 rather than BF16 crosses the interconnect) and consumed by + # Humming as-is. + activation_group_shape: GroupShape | None = None + input_scale_group_size = getattr(input_schema, "input_scale_group_size", 0) or 0 + if ( + q_dtype is not None + and q_dtype.startswith("float8") + and input_scale_group_size == 128 + ): + q_dtype = _HUMMING_TO_QUANT_DTYPE.get(input_schema.a_dtype, FP8_DTYPE) + activation_group_shape = GroupShape(row=1, col=input_scale_group_size) + weight_scale_group_size = weight_schema.weight_scale_group_size weight_scale_group_size_n = weight_schema.weight_scale_group_size_n weight_group_shape: tuple[int, ...] = () @@ -638,6 +670,7 @@ def get_humming_moe_quant_config( quant_dtype=q_dtype, weight_dtype=str(weight_schema.b_dtype), weight_group_shape=weight_group_shape, + activation_group_shape=activation_group_shape, w1_scale=getattr(layer, "w13_weight_scale", None), w1_gscale=getattr(layer, "w13_weight_scale_2", None), w1_zp=getattr(layer, "w13_zero_point", None),