[Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Elvir Crnčević <elvircrn@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Elvir Crnčević
2026-08-25 10:21:00 -04:00
committed by GitHub
co-authored by Claude Opus 4.8 Tyler Michael Smith mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9c8e90eb26
commit ca29cd48ba
17 changed files with 1594 additions and 171 deletions
+401 -14
View File
@@ -1,9 +1,17 @@
#include <cuda.h> #include <cuda.h>
#include <cuda_runtime.h>
#include <torch/csrc/stable/tensor.h> #include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/util/Float8_e4m3fn.h>
#ifndef USE_ROCM
#include <cuda_fp8.h>
#endif
#include <cmath> #include <cmath>
#include <type_traits>
#include "../cuda_compat.h" #include "../cuda_compat.h"
#include "async_util.cuh"
#include "cuda_vec_utils.cuh" #include "cuda_vec_utils.cuh"
#include "dispatch_utils.h" #include "dispatch_utils.h"
#include "torch_utils.h" #include "torch_utils.h"
@@ -471,15 +479,31 @@ __global__ void swigluoai_and_mul_kernel(
// gate_out = beta * tanh(gate / beta) * sigmoid(gate) // gate_out = beta * tanh(gate / beta) * sigmoid(gate)
// up_out = (linear_beta > 0) ? linear_beta * tanh(up / linear_beta) : up // up_out = (linear_beta > 0) ? linear_beta * tanh(up / linear_beta) : up
// out = gate_out * up_out // out = gate_out * up_out
// Compute is done in fp32 and written straight to `out` -- no intermediate __device__ __forceinline__ float situ_tanh(float x) { return tanhf(x); }
// tensors and no full-tensor fp32 upcast (the pure-torch forward_native
// allocated ~8 fp32 temporaries per call, which blows up MoE profiling). // Kimi-K3 SITU params; baked into the fused LDG kernel to fold at compile time.
static constexpr float SITU_BETA = 4.0f;
static constexpr float SITU_LINEAR_BETA = 25.0f;
__device__ __forceinline__ float situ_activation(float g, float u, float beta,
float linear_beta,
bool clamp_up, float inv_beta,
float inv_linear_beta) {
// sigmoid(g) == (1 + tanh(g/2)) / 2.
const float gate_out =
(0.5f * beta) * situ_tanh(g * inv_beta) * (1.0f + situ_tanh(g * 0.5f));
const float up_out =
clamp_up ? linear_beta * situ_tanh(u * inv_linear_beta) : u;
return gate_out * up_out;
}
template <typename scalar_t> template <typename scalar_t>
__global__ void situ_and_mul_kernel( __global__ void situ_and_mul_kernel(
scalar_t* __restrict__ out, // [..., d] scalar_t* __restrict__ out, // [..., d]
const scalar_t* __restrict__ input, // [..., 2, d] const scalar_t* __restrict__ input, // [..., 2, d]
const int d, const float beta, const float linear_beta) { const int d, const float beta, const float linear_beta,
const int64_t* __restrict__ valid_rows_ptr) {
const int64_t row = blockIdx.x; const int64_t row = blockIdx.x;
if (valid_rows_ptr != nullptr && row >= *valid_rows_ptr) return;
const scalar_t* gate_ptr = input + row * 2 * d; const scalar_t* gate_ptr = input + row * 2 * d;
const scalar_t* up_ptr = gate_ptr + d; const scalar_t* up_ptr = gate_ptr + d;
scalar_t* out_ptr = out + row * d; scalar_t* out_ptr = out + row * d;
@@ -489,10 +513,258 @@ __global__ void situ_and_mul_kernel(
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) { for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
const float g = (float)VLLM_LDG(&gate_ptr[idx]); const float g = (float)VLLM_LDG(&gate_ptr[idx]);
const float u = (float)VLLM_LDG(&up_ptr[idx]); const float u = (float)VLLM_LDG(&up_ptr[idx]);
const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g)); out_ptr[idx] = (scalar_t)situ_activation(g, u, beta, linear_beta, clamp_up,
const float up_out = inv_beta, inv_linear_beta);
clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u; }
out_ptr[idx] = (scalar_t)(gate_out * up_out); }
// Match Humming's hardware FP8 conversion. c10::Float8_e4m3fn's software cast
// can round ties differently from `cvt.rn.satfinite.e4m3.f32`.
__device__ __forceinline__ c10::Float8_e4m3fn cvt_fp8_hw(float x) {
#if !defined(USE_ROCM) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
return c10::Float8_e4m3fn(__nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3),
c10::Float8_e4m3fn::from_bits());
#else
return static_cast<c10::Float8_e4m3fn>(x);
#endif
}
template <typename fp8_type>
__device__ __forceinline__ fp8_type quant_to_fp8(float val, float inv_scale,
float fp8_max) {
float x = val * inv_scale;
x = fmaxf(-fp8_max, fminf(x, fp8_max));
if constexpr (std::is_same_v<fp8_type, c10::Float8_e4m3fn>) {
return cvt_fp8_hw(x);
} else {
return static_cast<fp8_type>(x);
}
}
__device__ __forceinline__ float warp_reduce_max(float v) {
#pragma unroll
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) {
v = fmaxf(v, VLLM_SHFL_XOR_SYNC(v, offset));
}
return v;
}
__device__ __forceinline__ float warp_reduce_absmax(float v) {
return warp_reduce_max(v);
}
template <typename scalar_t, typename fp8_type, int THREADS, int NUM_STAGES,
int GROUP_SIZE, int GRID_DIM, int D>
__global__ void situ_and_mul_quant_group_pipelined_kernel(
fp8_type* __restrict__ out, float* __restrict__ scale_out,
const scalar_t* __restrict__ input, const int64_t num_rows,
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
const int tid = threadIdx.x;
const int warp_id = tid / WARP_SIZE;
const int lane_id = tid % WARP_SIZE;
static constexpr int NUM_WARPS = THREADS / WARP_SIZE;
static constexpr int NUM_GROUPS = D / GROUP_SIZE;
const int64_t row_bound =
valid_rows_ptr != nullptr
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
: num_rows;
static_assert(NUM_GROUPS % 4 == 0,
"float4 scale fill needs NUM_GROUPS % 4 == 0");
static constexpr int NG4 = NUM_GROUPS / 4;
const int64_t pad_start4 = row_bound * NG4;
const int64_t pad_end4 = num_rows * NG4;
float4* scale4 = reinterpret_cast<float4*>(scale_out);
constexpr float4 ones{1.0f, 1.0f, 1.0f, 1.0f};
for (int64_t i = pad_start4 + (int64_t)blockIdx.x * THREADS + tid;
i < pad_end4; i += (int64_t)GRID_DIM * THREADS) {
scale4[i] = ones;
}
if constexpr (sizeof(scalar_t) == 2) {
static constexpr int LD_ELTS = 16 / sizeof(scalar_t);
static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE;
static constexpr int STAGE_ELTS = 2 * GROUP_SIZE;
extern __shared__ __align__(16) unsigned char smem_raw[];
scalar_t* warp_smem = reinterpret_cast<scalar_t*>(smem_raw) +
(size_t)warp_id * NUM_STAGES * STAGE_ELTS;
static constexpr float beta = SITU_BETA;
static constexpr float linear_beta = SITU_LINEAR_BETA;
static constexpr bool clamp_up = linear_beta > 0.0f;
static constexpr float inv_beta = 1.0f / beta;
static constexpr float inv_linear_beta =
clamp_up ? 1.0f / linear_beta : 0.0f;
static constexpr float FP8_MAX =
std::is_same_v<fp8_type, c10::Float8_e4m3fn> ? 448.0f : 224.0f;
static constexpr float inv_fp8_max = 1.0f / FP8_MAX;
static_assert(
NUM_GROUPS % NUM_WARPS == 0,
"constexpr num_iters requires groups evenly split across warps");
static constexpr int num_iters = NUM_GROUPS / NUM_WARPS;
const bool up_half = lane_id >= WARP_SIZE / 2;
const int lane_l = up_half ? lane_id - WARP_SIZE / 2 : lane_id;
const int lane_src_off = (up_half ? D : 0) + lane_l * LD_ELTS;
const int lane_dst_off = (up_half ? GROUP_SIZE : 0) + lane_l * LD_ELTS;
static constexpr int warp_stride = NUM_WARPS * GROUP_SIZE;
const scalar_t* src_ptr = input + warp_id * GROUP_SIZE + lane_src_off;
static constexpr int src_row_stride = GRID_DIM * 2 * D;
const scalar_t* row_src = src_ptr + blockIdx.x * 2 * D;
uint32_t* out_ptr =
reinterpret_cast<uint32_t*>(out + warp_id * GROUP_SIZE) + lane_id;
static constexpr int out_row_stride = GRID_DIM * D / 4;
uint32_t* row_out = out_ptr + blockIdx.x * (D / 4);
float* scale_ptr = scale_out + warp_id;
static constexpr int scale_row_stride = GRID_DIM * NUM_GROUPS;
float* row_scale = scale_ptr + blockIdx.x * NUM_GROUPS;
#pragma unroll 1
for (int64_t row = blockIdx.x; row < row_bound; row += GRID_DIM,
row_src += src_row_stride, row_out += out_row_stride,
row_scale += scale_row_stride) {
const scalar_t* src = row_src;
auto issue_load = [&](int it, int slot) {
if (it < num_iters) {
cuda_async::cp_async_shared_global_16_cg(
warp_smem + (size_t)slot * STAGE_ELTS + lane_dst_off, src);
src += warp_stride;
}
cuda_async::cp_async_commit_group();
};
int load_slot = 0;
auto bump = [](int s) { return s + 1 == NUM_STAGES ? 0 : s + 1; };
#pragma unroll
for (int s = 0; s < NUM_STAGES - 1; s++) {
issue_load(s, load_slot);
load_slot = bump(load_slot);
}
uint32_t* out_st = row_out;
float* scale_st = row_scale;
int comp_slot = 0;
for (int it = 0; it < num_iters; it++) {
issue_load(it + NUM_STAGES - 1, load_slot);
load_slot = bump(load_slot);
cuda_async::cp_async_wait_group<NUM_STAGES - 1>();
const scalar_t* stage = warp_smem + (size_t)comp_slot * STAGE_ELTS;
comp_slot = bump(comp_slot);
static_assert(ELTS_PER_LANE == 4,
"expects GROUP_SIZE == 4 * WARP_SIZE");
union V {
float2 f2;
scalar_t s[ELTS_PER_LANE];
};
const float2* gate2 = reinterpret_cast<const float2*>(stage);
const float2* up2 = reinterpret_cast<const float2*>(stage + GROUP_SIZE);
const V gv{gate2[lane_id]};
const V uv{up2[lane_id]};
const scalar_t* gs = gv.s;
const scalar_t* us = uv.s;
float acts[ELTS_PER_LANE];
float thread_max = 0.0f;
#pragma unroll
for (int e = 0; e < ELTS_PER_LANE; e++) {
acts[e] = (float)(scalar_t)situ_activation(
(float)gs[e], (float)us[e], beta, linear_beta, clamp_up, inv_beta,
inv_linear_beta);
thread_max = fmaxf(thread_max, fabsf(acts[e]));
}
const float absmax = fmaxf(warp_reduce_absmax(thread_max), 1e-30f);
const float scale = absmax * inv_fp8_max;
if (lane_id == 0) *scale_st = scale;
scale_st += NUM_WARPS;
const float inv_scale = __fdividef(1.0f, scale);
union O {
uint32_t u;
fp8_type f[ELTS_PER_LANE];
};
O o;
#pragma unroll
for (int e = 0; e < ELTS_PER_LANE; e++) {
o.f[e] = quant_to_fp8<fp8_type>(acts[e], inv_scale, FP8_MAX);
}
out_st[0] = o.u;
out_st += NUM_WARPS * GROUP_SIZE / 4;
}
cuda_async::cp_async_wait_group<0>();
}
}
}
// Scalar fallback for the group path (odd d or the fp32 dispatch branch).
template <typename scalar_t, typename fp8_type, int GROUP_SIZE>
__global__ void situ_and_mul_quant_group_scalar_kernel(
fp8_type* __restrict__ out, // [num_tokens, d]
float* __restrict__ scale_out, // [num_tokens, num_groups]
const scalar_t* __restrict__ input, // [num_tokens, 2, d]
const int d, const int num_groups, const float beta,
const float linear_beta, const int64_t num_rows,
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE;
const int tid = threadIdx.x;
const int warp_id = tid / WARP_SIZE;
const int lane_id = tid % WARP_SIZE;
const int num_warps = blockDim.x / WARP_SIZE;
const int64_t row_bound =
valid_rows_ptr != nullptr
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
: num_rows;
const bool clamp_up = linear_beta > 0.0f;
const float inv_beta = __fdividef(1.0f, beta);
const float inv_linear_beta = clamp_up ? __fdividef(1.0f, linear_beta) : 0.0f;
static constexpr float FP8_MAX =
std::is_same_v<fp8_type, c10::Float8_e4m3fn> ? 448.0f : 224.0f;
static constexpr float inv_fp8_max = 1.0f / FP8_MAX;
for (int64_t row = blockIdx.x; row < row_bound; row += gridDim.x) {
const scalar_t* gate_ptr = input + row * 2 * (int64_t)d;
const scalar_t* up_ptr = gate_ptr + d;
fp8_type* out_ptr = out + row * (int64_t)d;
float* scale_row = scale_out + row * (int64_t)num_groups;
for (int g = warp_id; g < num_groups; g += num_warps) {
const int base = g * GROUP_SIZE;
float acts[ELTS_PER_LANE];
float thread_max = 0.0f;
#pragma unroll
for (int e = 0; e < ELTS_PER_LANE; e++) {
const int idx = base + e * WARP_SIZE + lane_id;
const float gv = (float)VLLM_LDG(&gate_ptr[idx]);
const float uv = (float)VLLM_LDG(&up_ptr[idx]);
acts[e] = (float)(scalar_t)situ_activation(
gv, uv, beta, linear_beta, clamp_up, inv_beta, inv_linear_beta);
thread_max = fmaxf(thread_max, fabsf(acts[e]));
}
const float absmax = fmaxf(warp_reduce_max(thread_max), 1e-30f);
const float scale = absmax * inv_fp8_max;
if (lane_id == 0) scale_row[g] = scale;
const float inv_scale = __fdividef(1.0f, scale);
#pragma unroll
for (int e = 0; e < ELTS_PER_LANE; e++) {
const int idx = base + e * WARP_SIZE + lane_id;
out_ptr[idx] = quant_to_fp8<fp8_type>(acts[e], inv_scale, FP8_MAX);
}
}
}
// Fill skipped padding-row scales with 1 so they don't feed NaN into w2.
const int64_t pad_start = row_bound * (int64_t)num_groups;
const int64_t pad_end = num_rows * (int64_t)num_groups;
for (int64_t i = pad_start + (int64_t)blockIdx.x * blockDim.x + tid;
i < pad_end; i += (int64_t)gridDim.x * blockDim.x) {
scale_out[i] = 1.0f;
} }
} }
@@ -519,10 +791,8 @@ __global__ void masked_situ_and_mul_kernel(
scalar_t* out_ptr = out + row * d; scalar_t* out_ptr = out + row * d;
const float g = (float)VLLM_LDG(&gate_ptr[idx]); const float g = (float)VLLM_LDG(&gate_ptr[idx]);
const float u = (float)VLLM_LDG(&up_ptr[idx]); const float u = (float)VLLM_LDG(&up_ptr[idx]);
const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g)); out_ptr[idx] = (scalar_t)situ_activation(g, u, beta, linear_beta, clamp_up,
const float up_out = inv_beta, inv_linear_beta);
clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u;
out_ptr[idx] = (scalar_t)(gate_out * up_out);
} }
} }
@@ -620,7 +890,8 @@ void swigluoai_and_mul(torch::stable::Tensor& out, // [..., d]
// through), matching SituAndMul(linear_beta=None) on the Python side. // through), matching SituAndMul(linear_beta=None) on the Python side.
void situ_and_mul(torch::stable::Tensor& out, // [..., d] void situ_and_mul(torch::stable::Tensor& out, // [..., d]
torch::stable::Tensor& input, // [..., 2 * d] torch::stable::Tensor& input, // [..., 2 * d]
double beta, double linear_beta) { double beta, double linear_beta,
std::optional<torch::stable::Tensor> valid_rows) {
int d = input.size(-1) / 2; int d = input.size(-1) / 2;
int64_t num_tokens = input.numel() / input.size(-1); int64_t num_tokens = input.numel() / input.size(-1);
if (num_tokens == 0) { if (num_tokens == 0) {
@@ -628,6 +899,14 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
} }
dim3 grid(num_tokens); dim3 grid(num_tokens);
dim3 block(std::min(d, 1024)); dim3 block(std::min(d, 1024));
const int64_t* valid_rows_ptr = nullptr;
if (valid_rows.has_value()) {
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
input.get_device_index(),
"situ_and_mul: valid_rows must be a CUDA tensor on the "
"same device as input");
valid_rows_ptr = valid_rows->const_data_ptr<int64_t>();
}
const torch::stable::accelerator::DeviceGuard device_guard( const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index()); input.get_device_index());
const cudaStream_t stream = get_current_cuda_stream(); const cudaStream_t stream = get_current_cuda_stream();
@@ -635,7 +914,7 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
input.scalar_type(), "situ_and_mul_kernel", [&] { input.scalar_type(), "situ_and_mul_kernel", [&] {
vllm::situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>( vllm::situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(), out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
d, (float)beta, (float)linear_beta); d, (float)beta, (float)linear_beta, valid_rows_ptr);
}); });
} }
@@ -663,6 +942,114 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
(float)beta, (float)linear_beta); (float)beta, (float)linear_beta);
}); });
} }
// Fused SITU activation and block-FP8 quantization for Humming w2.
// `valid_rows` is the int32 DeepEP token count (psum[-1]); rows = it * `topk`,
// so padding is excluded on-device and skipped rows receive scale 1.
void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
torch::stable::Tensor& scale, // [..., 1 or d/128]
torch::stable::Tensor& input, // [..., 2 * d]
double beta, double linear_beta, int64_t group_size,
std::optional<torch::stable::Tensor> valid_rows,
int64_t topk) {
STD_TORCH_CHECK(
out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn ||
out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fnuz,
"situ_and_mul_quant output must be FP8 e4m3");
STD_TORCH_CHECK(
input.scalar_type() == torch::headeronly::ScalarType::Half ||
input.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"situ_and_mul_quant input must be FP16 or BF16");
STD_TORCH_CHECK(scale.scalar_type() == torch::headeronly::ScalarType::Float,
"situ_and_mul_quant scale must be float32");
STD_TORCH_CHECK(input.is_cuda() && out.is_cuda() && scale.is_cuda() &&
out.get_device_index() == input.get_device_index() &&
scale.get_device_index() == input.get_device_index(),
"situ_and_mul_quant: input, out and scale must be CUDA "
"tensors on the same device");
int d = input.size(-1) / 2;
int64_t num_tokens = input.numel() / input.size(-1);
if (num_tokens == 0) {
return;
}
STD_TORCH_CHECK(out.size(-1) == d && out.numel() == num_tokens * (int64_t)d,
"situ_and_mul_quant: out shape must be [num_tokens, d]");
const int32_t* valid_rows_ptr = nullptr;
if (valid_rows.has_value()) {
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
input.get_device_index(),
"situ_and_mul_quant: valid_rows must be a CUDA tensor on "
"the same device as input");
STD_TORCH_CHECK(
valid_rows->scalar_type() == torch::headeronly::ScalarType::Int,
"situ_and_mul_quant: valid_rows must be int32 (psum count)");
valid_rows_ptr = valid_rows->const_data_ptr<int32_t>();
}
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
static constexpr int BLOCKS_PER_SM = 8; // matches kernel __launch_bounds__
static constexpr int SM_COUNT = 132; // H200 (GH100, 132 SMs)
static constexpr int GRID_DIM = SM_COUNT * BLOCKS_PER_SM;
STD_TORCH_CHECK(group_size == 128,
"situ_and_mul_quant: only group_size 128 (block-FP8) "
"supported, got ",
group_size);
STD_TORCH_CHECK(d % group_size == 0, "situ_and_mul_quant: d (", d,
") must be divisible by group_size ", group_size);
{
const int num_groups = d / (int)group_size;
STD_TORCH_CHECK(scale.size(-1) == num_groups &&
scale.numel() == num_tokens * (int64_t)num_groups,
"situ_and_mul_quant: scale shape must be "
"[num_tokens, d/group_size]");
dim3 grid(GRID_DIM);
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "situ_and_mul_quant_group_kernel", [&] {
VLLM_STABLE_DISPATCH_FP8_TYPES(
out.scalar_type(), "situ_and_mul_quant_group_kernel_fp8", [&] {
#ifndef USE_ROCM
// The pipelined kernel's float2-per-lane geometry assumes a
// 32-lane warp (GROUP_SIZE == 4 * WARP_SIZE); on HIP (64-lane)
// fall back to the WARP_SIZE-generic scalar kernel.
constexpr int THREADS = 256;
constexpr int SITU_D = 3072; // Kimi-K3 fused w2 input dim
if constexpr (sizeof(scalar_t) == 2) {
if (d == SITU_D && (float)beta == vllm::SITU_BETA &&
(float)linear_beta == vllm::SITU_LINEAR_BETA) {
constexpr int D = SITU_D;
constexpr int GROUP_STAGES = 4;
constexpr int NUM_WARPS = THREADS / 32;
constexpr int STAGE_ELTS = 2 * 128;
dim3 block(THREADS);
size_t smem_bytes = (size_t)NUM_WARPS * GROUP_STAGES *
STAGE_ELTS * sizeof(scalar_t);
vllm::situ_and_mul_quant_group_pipelined_kernel<
scalar_t, fp8_t, THREADS, GROUP_STAGES, 128, GRID_DIM,
D><<<grid, block, smem_bytes, stream>>>(
out.mutable_data_ptr<fp8_t>(),
scale.mutable_data_ptr<float>(),
input.const_data_ptr<scalar_t>(), num_tokens,
valid_rows_ptr, topk);
return;
}
}
#endif
const int num_warps = std::min(num_groups, 1024 / WARP_SIZE);
dim3 block(num_warps * WARP_SIZE);
vllm::situ_and_mul_quant_group_scalar_kernel<scalar_t, fp8_t,
128>
<<<grid, block, 0, stream>>>(
out.mutable_data_ptr<fp8_t>(),
scale.mutable_data_ptr<float>(),
input.const_data_ptr<scalar_t>(), d, num_groups,
(float)beta, (float)linear_beta, num_tokens,
valid_rows_ptr, topk);
});
});
}
}
namespace vllm { namespace vllm {
// Element-wise activation kernel template. // Element-wise activation kernel template.
+10 -2
View File
@@ -526,8 +526,16 @@ void fatrelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
double threshold); double threshold);
void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
double alpha = 1.702, double limit = 7.0); double alpha = 1.702, double limit = 7.0);
void situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, void situ_and_mul(
double beta = 1.0, double linear_beta = -1.0); torch::stable::Tensor& out, torch::stable::Tensor& input, double beta = 1.0,
double linear_beta = -1.0,
std::optional<torch::stable::Tensor> valid_rows = std::nullopt);
void situ_and_mul_quant(
torch::stable::Tensor& out, torch::stable::Tensor& scale,
torch::stable::Tensor& input, double beta = 1.0, double linear_beta = -1.0,
int64_t group_size = 0,
std::optional<torch::stable::Tensor> valid_rows = std::nullopt,
int64_t topk = 1);
void masked_situ_and_mul(torch::stable::Tensor& out, void masked_situ_and_mul(torch::stable::Tensor& out,
torch::stable::Tensor& input, torch::stable::Tensor& input,
const torch::stable::Tensor& expert_num_tokens, const torch::stable::Tensor& expert_num_tokens,
+10 -1
View File
@@ -611,7 +611,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
// SituGLU implementation used in Kimi models. // SituGLU implementation used in Kimi models.
ops.def( ops.def(
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float " "situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
"linear_beta=-1.0) -> ()"); "linear_beta=-1.0, Tensor? valid_rows=None) -> ()");
// Fused SituGLU activation + dynamic FP8 quantization for the Humming w2 path
// (writes the fp8 down input and its float32 scale). group_size=0 ->
// per-token scale [.., 1]; group_size=128 -> k-major block-FP8 scale [..,
// d/128].
ops.def(
"situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, "
"float beta=1.0, float linear_beta=-1.0, int group_size=0, "
"Tensor? valid_rows=None, int topk=1) -> ()");
ops.def( ops.def(
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor " "masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()"); "expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
@@ -830,6 +838,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
ops.impl("fatrelu_and_mul", TORCH_BOX(&fatrelu_and_mul)); ops.impl("fatrelu_and_mul", TORCH_BOX(&fatrelu_and_mul));
ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul)); ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul));
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul)); ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant));
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul)); ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
ops.impl("gelu_new", TORCH_BOX(&gelu_new)); ops.impl("gelu_new", TORCH_BOX(&gelu_new));
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast)); ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
@@ -0,0 +1,11 @@
model_name: "nm-testing/Qwen3-30B-A3B-MXFP4A16"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--quantization humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3","group_size":128}'
@@ -1,5 +1,6 @@
gpt-oss-20b-humming-act-fp8.yaml gpt-oss-20b-humming-act-fp8.yaml
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8-block.yaml
Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
Qwen3-0.6B-MXFP8-humming-act-fp8.yaml Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
+1
View File
@@ -1408,6 +1408,7 @@ def test_humming_delegates_to_instance_activation():
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE, activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
input=input, input=input,
output=output, output=output,
valid_rows=None,
) )
+406
View File
@@ -0,0 +1,406 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the fused MoE topk weight-and-reduce kernel.
Run `pytest tests/kernels/moe/test_moe_fused_mul_sum.py`.
"""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.moe_fused_mul_sum import (
moe_fused_mul_sum,
)
from vllm.utils.torch_utils import set_random_seed
NUM_TOKENS = [1, 17, 256]
TOP_KS = [2, 8]
HIDDEN_SIZE = 512
# One BLOCK_K tile, a partial trailing tile, and Kimi-K3's real dims
# (moe_intermediate_size 3072, hidden_size 7168) -- exercises the per-tile loop
# and its tail mask (BLOCK_K caps at 512 for <=2-byte dtypes).
HIDDEN_SIZES = [512, 1000, 3072, 7168]
DTYPES = [torch.bfloat16, torch.float32]
NUM_EXPERTS = 16
def _reference(
inputs: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
) -> torch.Tensor:
valid = topk_ids >= 0
if expert_map is not None:
valid &= expert_map[topk_ids.clamp(min=0)] >= 0
weights = torch.where(valid, topk_weights, torch.zeros_like(topk_weights))
return (inputs.float() * weights.float().unsqueeze(-1)).sum(dim=1)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize("use_expert_map", [True, False])
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_invalid_slots_excluded(num_tokens: int, top_k: int, use_expert_map: bool):
"""Slots marked -1 must not contribute, and must not be dereferenced.
The expert GEMM never writes rows for topk slots whose expert id is -1
(a non-local expert under EP, or an all2all padding row), so those rows
hold stale workspace data. Filling them with NaN here means any failure to
mask them shows up as NaN in the output rather than a small numeric drift.
Indexing `expert_map` with -1 is also an out-of-bounds read.
"""
set_random_seed(0)
device = "cuda"
inputs = torch.randn(
num_tokens, top_k, HIDDEN_SIZE, dtype=torch.bfloat16, device=device
)
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
topk_ids = torch.randint(
0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
)
# Mark roughly half the slots invalid and poison the rows behind them.
invalid = torch.rand(num_tokens, top_k, device=device) < 0.5
topk_ids[invalid] = -1
inputs[invalid] = float("nan")
expert_map = None
if use_expert_map:
# Second half of the experts is non-local, so valid ids can still map
# to -1 -- exercising the second masking condition.
expert_map = torch.full((NUM_EXPERTS,), -1, dtype=torch.int32, device=device)
local = NUM_EXPERTS // 2
expert_map[:local] = torch.arange(local, dtype=torch.int32, device=device)
inputs[(topk_ids >= 0) & (expert_map[topk_ids.clamp(min=0)] < 0)] = float("nan")
# The _reference implementation zeros out outputs. We need to match that here,
# otherwise the test fails.
outputs = torch.zeros(num_tokens, HIDDEN_SIZE, dtype=torch.bfloat16, device=device)
out = moe_fused_mul_sum(
inputs=inputs,
topk_weights=topk_weights,
outputs=outputs,
topk_ids=topk_ids,
expert_map=expert_map,
)
assert not out.isnan().any(), "invalid slots leaked into the reduction"
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map)
torch.testing.assert_close(out.float(), ref, atol=1e-2, rtol=1e-2)
def _local_expert_map(device: str) -> torch.Tensor:
"""First half of the experts is local, second half maps to -1 (non-local)."""
expert_map = torch.full((NUM_EXPERTS,), -1, dtype=torch.int32, device=device)
local = NUM_EXPERTS // 2
expert_map[:local] = torch.arange(local, dtype=torch.int32, device=device)
return expert_map
SENTINEL = 42.0 # exact in bf16; marks output the kernel must leave untouched
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_nonlocal_row_is_zeroed():
"""A real row whose positive ids all map to non-local experts is written 0.
Such a row is not padding (its ids are >= 0, so row_valid > 0 and the block
is not early-returned). Every slot is then masked out by expert_map, so the
accumulator stays 0 and 0 is stored -- the row must not be left holding stale
buffer data, because downstream reduction treats it as a real token.
"""
set_random_seed(0)
device = "cuda"
num_tokens, top_k = 17, 8
local = NUM_EXPERTS // 2
expert_map = _local_expert_map(device)
# Poison inputs so any failure to mask the non-local slots surfaces as NaN.
inputs = torch.full(
(num_tokens, top_k, HIDDEN_SIZE),
float("nan"),
dtype=torch.bfloat16,
device=device,
)
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
# Every id lands in the non-local half [local, NUM_EXPERTS).
topk_ids = torch.randint(
local, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
)
outputs = torch.full(
(num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device
)
out = moe_fused_mul_sum(
inputs=inputs,
topk_weights=topk_weights,
outputs=outputs,
topk_ids=topk_ids,
expert_map=expert_map,
)
assert not out.isnan().any(), "non-local row leaked NaN into the output"
torch.testing.assert_close(out.float(), torch.zeros_like(out.float()))
@pytest.mark.parametrize("use_expert_map", [True, False])
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_padding_row_left_untouched(use_expert_map: bool):
"""An all -1 padding row is skipped: its output row is left untouched.
This is the CUDA-graph decode contract -- padding rows past num_recv are
never read downstream, so the kernel elides them (whole-padding blocks
early-return) rather than zeroing. The skip is gated on topk_ids alone, so
it must hold whether or not expert_map is passed (the humming path passes
None); we pin both.
"""
set_random_seed(0)
device = "cuda"
num_tokens, top_k = 17, 8
expert_map = _local_expert_map(device) if use_expert_map else None
inputs = torch.full(
(num_tokens, top_k, HIDDEN_SIZE),
float("nan"),
dtype=torch.bfloat16,
device=device,
)
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
topk_ids = torch.full((num_tokens, top_k), -1, dtype=torch.int32, device=device)
outputs = torch.full(
(num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device
)
out = moe_fused_mul_sum(
inputs=inputs,
topk_weights=topk_weights,
outputs=outputs,
topk_ids=topk_ids,
expert_map=expert_map,
)
assert torch.all(out == SENTINEL), "padding row output was modified"
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_nonlocal_and_padding_adjacent():
"""A non-local real row next to an all -1 padding row.
Each CTA owns one token row: the non-local row must be zeroed (its slots are
all masked by expert_map) while the padding row beside it is left untouched.
Guards against a change that lets one row's fate leak into its neighbour.
"""
set_random_seed(0)
device = "cuda"
num_tokens, top_k = 17, 8
local = NUM_EXPERTS // 2
expert_map = _local_expert_map(device)
inputs = torch.full(
(num_tokens, top_k, HIDDEN_SIZE),
float("nan"),
dtype=torch.bfloat16,
device=device,
)
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
# All rows are non-local real rows except row 1, an all -1 padding row.
topk_ids = torch.randint(
local, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
)
topk_ids[1] = -1
outputs = torch.full(
(num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device
)
out = moe_fused_mul_sum(
inputs=inputs,
topk_weights=topk_weights,
outputs=outputs,
topk_ids=topk_ids,
expert_map=expert_map,
)
padding = torch.zeros(num_tokens, dtype=torch.bool, device=device)
padding[1] = True
assert torch.all(out[1] == SENTINEL), "padding row beside a real row was modified"
written = out[~padding].float()
assert not written.isnan().any(), "non-local row leaked NaN into the output"
torch.testing.assert_close(written, torch.zeros_like(written))
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_weighted_sum_matches_reference(
num_tokens: int, top_k: int, hidden_size: int, dtype: torch.dtype
):
"""Weighted sum over live slots matches the reference across hidden tiles.
Covers the deployed humming path (globalized ids, expert_map=None): a random
-1 mix with NaN-poisoned dead slots, over hidden sizes that span a single
tile, a partial trailing tile, and many tiles, in both bf16 and fp32.
"""
set_random_seed(0)
device = "cuda"
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
topk_ids = torch.randint(
0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
)
invalid = torch.rand(num_tokens, top_k, device=device) < 0.5
topk_ids[invalid] = -1
inputs[invalid] = float("nan")
# All-padding rows early-return untouched, so pre-zero to match the reference.
outputs = torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device)
out = moe_fused_mul_sum(
inputs=inputs,
topk_weights=topk_weights,
outputs=outputs,
topk_ids=topk_ids,
expert_map=None,
)
assert not out.isnan().any(), "invalid slots leaked into the reduction"
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map=None)
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_no_topk_ids_sums_all_slots(dtype: torch.dtype):
"""With topk_ids=None every slot is live: plain weighted sum over top_k."""
set_random_seed(0)
device = "cuda"
num_tokens, top_k, hidden_size = 17, 8, 1000
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
out = moe_fused_mul_sum(inputs=inputs, topk_weights=topk_weights)
ref = (inputs.float() * topk_weights.float().unsqueeze(-1)).sum(dim=1)
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
@pytest.mark.parametrize("hidden_size", [512, 7168])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_expert_map_is_required_for_nonlocal_ids(hidden_size: int, dtype: torch.dtype):
"""expert_map is load-bearing, not redundant: dropping it leaks non-local ids.
A positive id whose expert is non-local marks a slot the GEMM never wrote, so
its `inputs` row is stale. `topk_ids >= 0` alone cannot tell it apart from a
real slot -- only expert_map can. This pins the contract the humming path
relies on (globalized ids are still routed through expert_map as a safety
net): with the map the stale slot is masked; without it, it corrupts the sum.
Regression guard against re-dropping expert_map as an optimization.
"""
set_random_seed(0)
device = "cuda"
num_tokens, top_k = 8, 8
local = NUM_EXPERTS // 2
expert_map = _local_expert_map(device)
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
# Each row keeps at least one local slot (so the mapped output is finite) and
# at least one non-local slot whose stale row is poisoned with NaN.
half = top_k // 2
topk_ids = torch.empty(num_tokens, top_k, dtype=torch.int32, device=device)
topk_ids[:, :half] = torch.randint(
0, local, (num_tokens, half), dtype=torch.int32, device=device
)
topk_ids[:, half:] = torch.randint(
local, NUM_EXPERTS, (num_tokens, top_k - half), dtype=torch.int32, device=device
)
inputs[:, half:] = float("nan")
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
mapped = moe_fused_mul_sum(
inputs=inputs,
topk_weights=topk_weights,
outputs=torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device),
topk_ids=topk_ids,
expert_map=expert_map,
)
assert not mapped.isnan().any(), "expert_map failed to mask stale non-local slot"
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map)
torch.testing.assert_close(mapped.float(), ref, atol=atol, rtol=rtol)
# Same call without the map: the non-local slot is summed and the NaN leaks.
unmapped = moe_fused_mul_sum(
inputs=inputs,
topk_weights=topk_weights,
outputs=torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device),
topk_ids=topk_ids,
expert_map=None,
)
assert unmapped.isnan().any(), "expert_map is not redundant: dropping it must leak"
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_num_valid_tokens_bounds_stale_padding(dtype: torch.dtype):
"""Rows past num_valid_tokens are left untouched, even with stale ids/NaN.
Under cudagraph decode the grid is the static padded token count and the
tail rows past num_recv hold stale (non -1) ids and stale NaN in `inputs`
from a prior replay -- the -1 mask alone cannot spot them. Bounding on
num_valid_tokens must skip the tail before any load so its NaN never reaches
the output; without the bound it is summed and a masked finalize can then
propagate the NaN. Regression guard for the decode accuracy fix.
"""
set_random_seed(0)
device = "cuda"
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 512
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
# All ids valid (>= 0): the stale tail is indistinguishable by the -1 mask.
topk_ids = torch.randint(
0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
)
inputs[num_recv:] = float("nan") # stale down_output behind the padding tail
num_valid = torch.tensor([num_recv], dtype=torch.int32, device=device)
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
bounded = moe_fused_mul_sum(
inputs=inputs,
topk_weights=topk_weights,
outputs=torch.full(
(num_tokens, hidden_size), SENTINEL, dtype=dtype, device=device
),
topk_ids=topk_ids,
num_valid_tokens=num_valid,
)
assert torch.all(bounded[num_recv:] == SENTINEL), (
"padding tail past num_recv written"
)
assert not bounded[:num_recv].isnan().any(), "real rows leaked NaN"
ref = _reference(
inputs[:num_recv].nan_to_num(),
topk_weights[:num_recv],
topk_ids[:num_recv],
expert_map=None,
)
torch.testing.assert_close(bounded[:num_recv].float(), ref, atol=atol, rtol=rtol)
# Without the bound the stale tail is summed and leaks NaN into its rows.
unbounded = moe_fused_mul_sum(
inputs=inputs,
topk_weights=topk_weights,
outputs=torch.full(
(num_tokens, hidden_size), SENTINEL, dtype=dtype, device=device
),
topk_ids=topk_ids,
)
assert unbounded[num_recv:].isnan().any(), "num_valid_tokens bound is load-bearing"
@@ -0,0 +1,224 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the persistent cp.async-pipelined SITU + block-FP8
quant kernel (``situ_and_mul_quant``).
Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference
activation + block-FP8 quant, drive random per-row scale magnitudes, exercise
the masked (``valid_rows``) contiguous layout, and compare per valid row.
The kernel computes SITU in fp32 and converts to fp8 with hardware rounding,
which torch cannot reproduce bit-exactly. It is therefore validated against the
*ideal* FP8 quantizer of the exact fp64 activation: the difference must stay
within the unavoidable FP8 block-quant error (within 1 fp8 code for ~all
elements, never off by more than 2).
"""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.activation import situ_and_mul_quant
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
fp8_dtype = current_platform.fp8_dtype()
DEVICE = current_platform.device_type
FP8_MAX = torch.finfo(fp8_dtype).max
# The kernel uses fp8 e4m3 conversion, so it is CUDA-only and needs SM89+
# (Ada/Hopper). There is no Triton fallback.
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not current_platform.has_device_capability(89),
reason="situ_and_mul_quant persistent kernel requires CUDA SM89+ (fp8 e4m3).",
)
# Kimi-K3 baked SITU params. Only (d=3072, beta=4.0, linear_beta=25.0,
# group_size=128, fp16/bf16) routes to the persistent cp.async-pipelined kernel;
# anything else falls back to the scalar group kernel.
SITU_BETA = 4.0
SITU_LINEAR_BETA = 25.0
SITU_D = 3072
GROUP_SIZE = 128
# (num_tokens, valid_rows). valid_rows=None => all rows valid. 2048/4096 rows
# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8).
CASES = [
(1, None),
(17, None),
(128, None),
(2048, None),
(4096, None),
(2048, 1500), # DeepEP v2 contiguous padding
(2048, 1),
(4096, 4095),
(512, 0), # whole tensor is padding
]
def token_random(num_tokens: int, twod: int, dtype: torch.dtype) -> torch.Tensor:
"""Per-row random exponent so groups span many scale magnitudes."""
base = torch.randn(num_tokens, twod, dtype=torch.float32, device=DEVICE)
exps = torch.randint(1, 13, (num_tokens, 1), device=DEVICE).float()
return (base * (2.0**exps)).to(dtype)
def ref_situ_block_fp8_quant(
inp: torch.Tensor, beta: float, linear_beta: float, group_size: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Ideal FP8 block-quant of the exact fp64 SITU activation.
The kernel rounds the activation to ``scalar_t`` before quantizing (it stores
``(float)(scalar_t)situ(...)``), so the reference does the same before taking
the per-group absmax.
"""
d = inp.shape[-1] // 2
gate, up = inp.double().chunk(2, dim=-1)
gate_out = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
up_out = linear_beta * torch.tanh(up / linear_beta) if linear_beta > 0 else up
act = (gate_out * up_out).to(inp.dtype).float()
m = act.shape[0]
ng = d // group_size
a = act.view(m, ng, group_size)
scale = a.abs().amax(dim=2).clamp_min(1e-30) / FP8_MAX
q = (a / scale.unsqueeze(2)).clamp(-FP8_MAX, FP8_MAX).to(fp8_dtype)
return q.view(m, d), scale
def fp8_code_dist(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
"""Signed-magnitude fp8 code distance (valid for e4m3fn and e4m3fnuz)."""
def ordered(x: torch.Tensor) -> torch.Tensor:
x = x.view(torch.uint8).to(torch.int32)
return torch.where((x >> 7) == 1, -(x & 0x7F), x & 0x7F)
return (ordered(a) - ordered(b)).abs()
@pytest.mark.parametrize("num_tokens,valid", CASES)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
@torch.inference_mode()
def test_situ_and_mul_quant_pipelined(
num_tokens: int, valid: int | None, dtype: torch.dtype
) -> None:
set_random_seed(42)
d = SITU_D
inp = token_random(num_tokens, 2 * d, dtype)
# Sentinels: padding rows must keep their bytes (out) and get scale 1.0.
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
scale = torch.full(
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
)
out_sentinel = out.clone()
valid_rows = (
None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE)
)
situ_and_mul_quant(
out,
scale,
inp,
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
valid_rows=valid_rows,
)
# Padding contract: skipped rows keep their bytes; their scale becomes 1.0.
if valid is not None and valid < num_tokens:
assert torch.equal(scale[valid:], torch.ones_like(scale[valid:]))
assert torch.equal(
out[valid:].view(torch.uint8), out_sentinel[valid:].view(torch.uint8)
)
rows = num_tokens if valid is None else valid
if rows == 0:
return
ref_q, ref_s = ref_situ_block_fp8_quant(
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
)
assert torch.isfinite(scale[:rows]).all()
assert not torch.isnan(out[:rows].float()).any()
# Scales: the kernel's fp32 SITU shifts the per-group absmax only
# marginally vs the fp64 reference.
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
# FP8 codes: must land within 1 code of the ideal quantizer for ~all
# elements, never off by more than 2.
dist = fp8_code_dist(out[:rows], ref_q)
assert dist.max().item() <= 2
assert (dist <= 1).float().mean().item() >= 0.99
@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)])
@torch.inference_mode()
def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
"""valid_rows is a token count; the kernel bounds rows at tokens*topk."""
set_random_seed(7)
d = SITU_D
num_tokens = tokens * topk + 37 # padding tail past the valid rows
inp = token_random(num_tokens, 2 * d, torch.bfloat16)
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
scale = torch.full(
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
)
out_sentinel = out.clone()
valid_rows = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
situ_and_mul_quant(
out,
scale,
inp,
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
valid_rows=valid_rows,
topk=topk,
)
rows = tokens * topk
assert torch.equal(scale[rows:], torch.ones_like(scale[rows:]))
assert torch.equal(
out[rows:].view(torch.uint8), out_sentinel[rows:].view(torch.uint8)
)
ref_q, ref_s = ref_situ_block_fp8_quant(
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
)
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
assert fp8_code_dist(out[:rows], ref_q).max().item() <= 2
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
@torch.inference_mode()
def test_situ_and_mul_quant_pipelined_deterministic(dtype: torch.dtype) -> None:
"""Two runs on the same input must be byte-identical."""
set_random_seed(0)
d = SITU_D
num_tokens = 2048
inp = token_random(num_tokens, 2 * d, dtype)
def run() -> tuple[torch.Tensor, torch.Tensor]:
out = torch.empty(num_tokens, d, dtype=fp8_dtype, device=DEVICE)
scale = torch.empty(
num_tokens, d // GROUP_SIZE, dtype=torch.float32, device=DEVICE
)
situ_and_mul_quant(
out,
scale,
inp,
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
)
return out, scale
out1, s1 = run()
out2, s2 = run()
assert torch.equal(out1.view(torch.uint8), out2.view(torch.uint8))
assert torch.equal(s1, s2)
@@ -0,0 +1,122 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Humming MXFP4-weight GEMM paired with a block-FP8 activation.
The Humming kernel already implements MXFP4 weight (group-32 e8m0) with a
grouped FP8 activation via WGMMA software dequant -- see the grouped-fp8
`test_mxfp4.py` cases in the humming repo, which run on SM90 (H200). Enabling
it in vLLM comes down to two things, both exercised here:
1. A grouped FP8 activation must classify to a float32-scaled key
(kFp8Dynamic128Sym), not a uint8 microscale key. MX microscale
activations (group size 32) keep their e8m0 (uint8) scale.
2. (kMxfp4Static, kFp8Dynamic128Sym) must be an allowed MoE
(weight, activation) pair.
"""
import pytest
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingExpertsBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTokenSym,
kMxfp4Static,
kMxfp8Dynamic,
)
from vllm.utils.import_utils import has_humming
# make_humming_moe_quant_config stores per-sublayer humming LayerConfigs but the
# config-plumbing asserts below never read them, so a placeholder satisfies the
# required arg without pulling in the humming package.
_PLACEHOLDER_HUMMING_CONFIGS = {"w13": None, "w2": None}
def test_mxfp4_weight_with_block_fp8_activation_is_supported():
"""MXFP4 weight + block-FP8 (group-128) activation is an allowed pair."""
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8Dynamic128Sym)
def test_mxfp4_weight_with_per_token_fp8_activation_still_supported():
"""The pre-existing per-token FP8 pairing must keep working."""
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8DynamicTokenSym)
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
@pytest.mark.parametrize(
"group_size,expected_key",
[
# per-token FP8 -> float32 per-token scale
(0, kFp8DynamicTokenSym),
# block FP8 -> float32 group-128 scale (the newly-fixed case)
(128, kFp8Dynamic128Sym),
# MXFP8 microscale -> e8m0 (uint8) group-32 scale (must be preserved)
(32, kMxfp8Dynamic),
],
)
def test_humming_fp8_input_schema_to_quant_key(group_size, expected_key):
from vllm.model_executor.layers.quantization.utils.humming_utils import (
_humming_input_schema_to_quant_key,
)
from vllm.utils.humming import HummingInputSchema
from vllm.utils.humming import dtypes as humming_dtypes
schema = HummingInputSchema(
a_dtype=humming_dtypes.float8e4m3,
input_scale_group_size=group_size,
)
assert _humming_input_schema_to_quant_key(schema) == expected_key
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
def test_humming_bf16_input_schema_is_unquantized():
from vllm.model_executor.layers.quantization.utils.humming_utils import (
_humming_input_schema_to_quant_key,
)
from vllm.utils.humming import HummingInputSchema
# No a_dtype -> unquantized (bf16/fp16) inputs -> None.
assert _humming_input_schema_to_quant_key(HummingInputSchema()) is None
def test_block_fp8_activation_quant_config_is_block_quantized():
"""A block-FP8 activation group shape yields a block-quantized MoE config
(FP8 dtype, [1, 128] block, not per-act-token). This is what makes the
DeepEP prepare/finalize step quantize activations to block FP8 *before* the
all-to-all dispatch instead of deferring to Humming -- and what makes
HummingExpertsBase.expects_unquantized_inputs return False."""
from vllm.model_executor.layers.quantization.utils.humming_utils import (
make_humming_moe_quant_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.platforms import current_platform
qc = make_humming_moe_quant_config(
quant_dtype=current_platform.fp8_dtype(),
weight_dtype="float4e2m1",
activation_group_shape=GroupShape(row=1, col=128),
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
)
assert qc.is_block_quantized
assert qc.block_shape == [1, 128]
assert not qc.per_act_token_quant
assert qc.quant_dtype == current_platform.fp8_dtype()
def test_default_activation_quant_config_defers_to_humming():
"""Without a block activation shape the config stays per-token (the deferred
path): Humming quantizes internally, preserving pre-existing behavior for
every non-block-FP8 scheme."""
from vllm.model_executor.layers.quantization.utils.humming_utils import (
make_humming_moe_quant_config,
)
from vllm.platforms import current_platform
qc = make_humming_moe_quant_config(
quant_dtype=current_platform.fp8_dtype(),
weight_dtype="float4e2m1",
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
)
assert not qc.is_block_quantized
assert qc.per_act_token_quant
@@ -201,6 +201,7 @@ def apply_moe_activation(
activation_config: ApplyMoEActivationConfig | None = None, activation_config: ApplyMoEActivationConfig | None = None,
topk_ids: torch.Tensor | None = None, topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None, expert_map: torch.Tensor | None = None,
valid_rows: torch.Tensor | None = None,
) -> torch.Tensor: ) -> torch.Tensor:
"""Apply MoE activation function. """Apply MoE activation function.
@@ -256,6 +257,7 @@ def apply_moe_activation(
-1.0 -1.0
if config.activation_situ_linear_beta is None if config.activation_situ_linear_beta is None
else config.activation_situ_linear_beta, else config.activation_situ_linear_beta,
valid_rows,
) )
elif activation == MoEActivation.SWIGLUOAI: elif activation == MoEActivation.SWIGLUOAI:
torch.ops._C.swigluoai_and_mul(output, input) torch.ops._C.swigluoai_and_mul(output, input)
@@ -286,3 +288,45 @@ def apply_moe_activation(
raise ValueError(f"Unsupported FusedMoe activation: {activation}") raise ValueError(f"Unsupported FusedMoe activation: {activation}")
return output return output
def situ_and_mul_quant(
output: torch.Tensor,
scale: torch.Tensor,
input: torch.Tensor,
*,
beta: float,
linear_beta: float | None,
group_size: int = 0,
valid_rows: torch.Tensor | None = None,
topk: int = 1,
) -> None:
"""Fused Kimi SITU activation + dynamic FP8 quantization.
Writes the quantized fp8 down-projection input into ``output`` [M, d] and its
float32 scale into ``scale`` (dequant = q * scale), fusing the SITU
activation with the FP8 quant that Humming's w2 GEMM would otherwise perform
in a separate pass -- this avoids materializing (and rounding through) the
bf16 activation_output buffer.
``group_size`` selects the quant granularity: ``0`` gives per-token scales
``scale`` [M, 1]; ``128`` gives k-major block-FP8 group scales ``scale``
[M, d // 128], matching humming ``quant_input(group_size=128, float32)``.
``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching
``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the
DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the
kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply
is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept
in this module so that every ``torch.ops._C.situ*`` call lives in one file.
"""
torch.ops._C.situ_and_mul_quant(
output,
scale,
input,
beta,
-1.0 if linear_beta is None else linear_beta,
group_size,
valid_rows,
topk,
)
@@ -525,7 +525,9 @@ def deepgemm_moe_permute(
expert_num_tokens = None expert_num_tokens = None
if expert_tokens_meta is not None: if expert_tokens_meta is not None:
expert_num_tokens = expert_tokens_meta.expert_num_tokens expert_num_tokens = expert_tokens_meta.expert_num_tokens
else: if expert_num_tokens is None:
# A cudagraph-decode carrier may hold only psum_recv_per_rank and leave
# expert_num_tokens None; count locally as in the meta-absent case.
expert_num_tokens = count_expert_num_tokens( expert_num_tokens = count_expert_num_tokens(
topk_ids, local_num_experts, expert_map topk_ids, local_num_experts, expert_map
) )
@@ -668,6 +668,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
assert hidden_states.dim() == 3 assert hidden_states.dim() == 3
assert expert_tokens_meta is not None assert expert_tokens_meta is not None
expert_num_tokens = expert_tokens_meta.expert_num_tokens expert_num_tokens = expert_tokens_meta.expert_num_tokens
assert expert_num_tokens is not None
num_local_experts = w1.size(0) num_local_experts = w1.size(0)
assert num_local_experts == w1.size(0), f"{num_local_experts} == {w1.size(0)}" assert num_local_experts == w1.size(0), f"{num_local_experts} == {w1.size(0)}"
@@ -119,6 +119,37 @@ def get_humming_moe_gemm_type() -> str:
return gemm_type return gemm_type
def _fixup_moe_tuning_config(tuning_config: list, max_k_block: int = 128) -> None:
"""Fix up each MoE tile in place: cap the K-block and widen warp-N.
- block_shape[2] (K-block) > ``max_k_block``: the driver rejects the TMA
descriptor at launch (CUDA_ERROR_MISALIGNED_ADDRESS). Cap at 128, which
Humming already uses for larger M.
- warp_shape[1] (warp-N) < 32: block-FP8 (group-128) activations route the
w13 (gate/up) GEMM to a tuning table that pins warp-N to 16, under-filling
the Hopper WGMMA N dimension and corrupting the GEMM output (gsm8k
0.94 -> 0.89). Widen to 32 whenever block_n % 32 == 0 -- on every tile,
not just the K-capped ones. w2 (down) already uses warp_n=32.
"""
logger.info_once("Attempting to override humming GEMM config")
for entry in tuning_config:
config = entry[2]
block_shape = config.get("block_shape")
if not (block_shape and len(block_shape) == 3):
continue
block_m, block_n, block_k = block_shape
logger.info_once(f"Overriding humming GEMM config. Previous config\n: {config}")
if block_k > max_k_block:
config["block_shape"] = [block_m, block_n, max_k_block]
warp_shape = config.get("warp_shape")
if warp_shape and warp_shape[1] < 32 and block_n % 32 == 0:
config["warp_shape"] = [warp_shape[0], 32, warp_shape[2]]
logger.info_once(f"Overridden humming GEMM config. Current config\n: {config}")
class HummingExpertsBase(mk.FusedMoEExpertsModular): class HummingExpertsBase(mk.FusedMoEExpertsModular):
def __init__( def __init__(
self, self,
@@ -170,6 +201,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
gemm_type=self.humming_gemm_type(), gemm_type=self.humming_gemm_type(),
) )
self.compute_config_str = json.dumps(self.compute_config) self.compute_config_str = json.dumps(self.compute_config)
# Fix up the heuristic-chosen tiles before freezing to JSON -- see
# _fixup_moe_tuning_config.
_fixup_moe_tuning_config(self.w13_tuning_config)
_fixup_moe_tuning_config(self.w2_tuning_config)
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config) self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config) self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
@@ -178,12 +213,17 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
sublayer_name: str, sublayer_name: str,
inputs: torch.Tensor, inputs: torch.Tensor,
quanted_input: torch.Tensor | None, quanted_input: torch.Tensor | None,
input_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]: ) -> tuple[torch.Tensor, torch.Tensor | None]:
from vllm.utils.humming import may_quant_input from vllm.utils.humming import may_quant_input
# input_scale is set for block-FP8 (group-128) activations that were
# quantized before the EP dispatch: may_quant_input then skips the
# redundant w13 quantization and forwards the pre-computed scale.
return may_quant_input( return may_quant_input(
self.humming_configs[sublayer_name], self.humming_configs[sublayer_name],
inputs=inputs, inputs=inputs,
input_scale=input_scale,
quanted_input=quanted_input, quanted_input=quanted_input,
) )
@@ -263,6 +303,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
(kMxfp4Static, kMxfp4Dynamic), (kMxfp4Static, kMxfp4Dynamic),
(kMxfp4Static, kMxfp8Dynamic), (kMxfp4Static, kMxfp8Dynamic),
(kMxfp4Static, kFp8DynamicTokenSym), (kMxfp4Static, kFp8DynamicTokenSym),
# MXFP4 weight (group-32 e8m0) with block-FP8 activation
# (group-128 float32). Runs via WGMMA software dequant, so it
# works on Hopper (SM90/H200) as well as Blackwell.
(kMxfp4Static, kFp8Dynamic128Sym),
(kNvfp4Static, None), (kNvfp4Static, None),
(kNvfp4Static, kFp8DynamicTokenSym), (kNvfp4Static, kFp8DynamicTokenSym),
(kMxfp8Static, None), (kMxfp8Static, None),
@@ -299,21 +343,43 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
and _is_supported_wna16_weight_key(weight_key) and _is_supported_wna16_weight_key(weight_key)
) )
def _prequantizes_dispatch_activation(self) -> bool:
"""
Whether the prepare/finalize step should quantize activations before
the (EP all-to-all) dispatch instead of leaving it to Humming.
This is enabled only for block-FP8 (group-128) activations: quantizing
to FP8 before dispatch sends FP8 rather than BF16 over the interconnect,
and Humming then consumes the pre-quantized FP8 + scale directly (see
the apply() methods, which forward the dispatch scale into
HummingMethod.may_quant_input, and may_quant_input itself, which is a
no-op when an input scale is already supplied). The scale layout
produced by vLLM's block-FP8 quantization ([M, K // 128] float32,
row-major) matches what the Humming WGMMA grouped GEMM expects.
"""
quant_config = self.quant_config
return (
quant_config.is_block_quantized
and quant_config.quant_dtype == current_platform.fp8_dtype()
)
@property @property
def expects_unquantized_inputs(self) -> bool: def expects_unquantized_inputs(self) -> bool:
""" """
Humming kernels handle input quantization internally in apply(). Whether the prepare/finalize step should defer input quantization to
the experts (by setting defer_input_quant=True and passing unquantized
inputs).
This property tells the prepare/finalize step to skip input Humming normally quantizes inputs internally via
quantization (by setting defer_input_quant=True) and pass HummingMethod.may_quant_input() in apply(), so we defer quantization
unquantized inputs to the experts. This prevents double (return True) to avoid quantizing twice -- once in prepare and once in
quantization: once in prepare and once in Humming's apply(). Humming's apply().
Returns: The exception is block-FP8 (group-128) activations, which are quantized
True to indicate that this expert expects unquantized inputs before the dispatch to save interconnect bandwidth (see
and will handle quantization internally. _prequantizes_dispatch_activation): for those we must NOT defer.
""" """
return True return not self._prequantizes_dispatch_activation()
@staticmethod @staticmethod
def _supports_current_device() -> bool: def _supports_current_device() -> bool:
@@ -585,6 +651,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
activation: MoEActivation, activation: MoEActivation,
output: torch.Tensor, output: torch.Tensor,
input: torch.Tensor, input: torch.Tensor,
valid_rows: torch.Tensor | None = None,
) -> None: ) -> None:
activation_config = self.activation_config activation_config = self.activation_config
if ( if (
@@ -601,8 +668,84 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
activation=activation, activation=activation,
input=input, input=input,
output=output, output=output,
valid_rows=valid_rows,
) )
def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool:
"""Whether the SITU activation + w2 quant can be fused into one kernel.
Fused only for k-major block-FP8 group-128 e4m3 with float32 scales --
the sole layout situ_and_mul_quant supports. A float32 as_dtype rules out
MXMMA (which uses e8m0 m-major scales), so a group-128 scale is
guaranteed k-major. Everything else (m-major MXFP8, other group sizes,
16-bit passthrough, non-SITU) falls back to the separate
situ_and_mul + quantize_input path.
"""
if activation != MoEActivation.SITU:
return False
w2cfg = self.humming_configs["w2"]
# Report the specific blocker once so a silent fallback to the unfused
# situ_and_mul + quant pair is never a mystery in a trace.
reason: str | None = None
if not (w2cfg.a_dtype.num_bits == 8 and str(w2cfg.a_dtype) == "float8e4m3"):
reason = f"w2 a_dtype is {w2cfg.a_dtype} (need float8e4m3)"
elif w2cfg.input_scale_group_size != 128:
reason = (
f"w2 input_scale_group_size is {w2cfg.input_scale_group_size} "
"(need 128)"
)
elif not (w2cfg.as_dtype is not None and str(w2cfg.as_dtype) == "float32"):
reason = f"w2 as_dtype is {w2cfg.as_dtype} (need float32, k-major)"
if reason is not None:
logger.warning_once(
"Humming fused SITU+FP8 quant disabled, using unfused "
"situ_and_mul + quant: %s",
reason,
)
return False
return True
def fused_situ_quant(
self,
gate_up_output: torch.Tensor,
quanted_down_input: torch.Tensor,
valid_rows: torch.Tensor | None,
topk: int = 1,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the fused SITU activation + FP8 quant for the w2 input.
Returns ``(quanted_down_input, input_scale)`` in the same layout the
unfused ``apply_activation`` + ``quantize_input("w2")`` pair produces: an
fp8 [M, d] tensor plus a k-major block-FP8 float32 [M, d // 128] group
scale. Only reached when ``fused_situ_quant_enabled`` (group_size 128).
"""
from vllm.model_executor.layers.fused_moe.activation import (
situ_and_mul_quant,
)
cfg = self.activation_config
assert cfg.activation_situ_beta is not None, (
"SITU requires activation_situ_beta from FusedMoEConfig"
)
group_size = self.humming_configs["w2"].input_scale_group_size
m, d = quanted_down_input.size(0), quanted_down_input.size(1)
input_scale = torch.empty(
(m, d // group_size),
dtype=torch.float32,
device=quanted_down_input.device,
)
situ_and_mul_quant(
quanted_down_input,
input_scale,
gate_up_output,
beta=cfg.activation_situ_beta,
linear_beta=cfg.activation_situ_linear_beta,
group_size=group_size,
valid_rows=valid_rows,
topk=topk,
)
return quanted_down_input, input_scale
class HummingIndexedExperts(HummingExpertsBase): class HummingIndexedExperts(HummingExpertsBase):
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
@@ -684,10 +827,12 @@ class HummingIndexedExperts(HummingExpertsBase):
""" """
Standard apply implementation for Humming indexed experts. Standard apply implementation for Humming indexed experts.
Humming performs activation quantization internally and consumes the Note: Humming kernels handle weights internally through the layer
weights supplied by the modular kernel interface. object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
Results are written to the output buffer supplied by the modular path (Humming quantizes the w13 input itself); for block-FP8 activations
kernel, which may alias workspace13. it carries the scale computed before dispatch, which is forwarded to
may_quant_input so Humming skips the redundant w13 quantization. The
output is written into workspace13 via the buffer management.
""" """
assert not apply_router_weight_on_input assert not apply_router_weight_on_input
@@ -710,6 +855,7 @@ class HummingIndexedExperts(HummingExpertsBase):
inputs, input_scale = self.quantize_input( inputs, input_scale = self.quantize_input(
"w13", "w13",
inputs=hidden_states, inputs=hidden_states,
input_scale=a1q_scale,
quanted_input=buffers.get("quanted_gate_up_input", None), quanted_input=buffers.get("quanted_gate_up_input", None),
) )
@@ -722,17 +868,45 @@ class HummingIndexedExperts(HummingExpertsBase):
**moe_kwargs1, **moe_kwargs1,
) )
self.apply_activation( # psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view.
activation=activation, # situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds
input=buffers["gate_up_output"], # on tokens, so both read this pointer directly -- no host cast/multiply.
output=buffers["activation_output"], valid_tokens = None
) topk = topk_ids.size(1)
if (
expert_tokens_meta is not None
and expert_tokens_meta.psum_recv_per_rank is not None
):
valid_tokens = expert_tokens_meta.psum_recv_per_rank[-1:]
inputs, input_scale = self.quantize_input( if self.fused_situ_quant_enabled(activation):
"w2", # Fused SITU + FP8 quant (per-token or block-FP8 group-128) straight
inputs=buffers["activation_output"], # into the w2 input, skipping the bf16 activation_output round-trip.
quanted_input=buffers.get("quanted_down_input", None), inputs, input_scale = self.fused_situ_quant(
) gate_up_output=buffers["gate_up_output"],
quanted_down_input=buffers["quanted_down_input"],
valid_rows=valid_tokens,
topk=topk,
)
else:
# Fallback situ_and_mul takes int64 row counts (tokens*topk).
valid_rows = (
valid_tokens.to(torch.int64) * topk
if valid_tokens is not None
else None
)
self.apply_activation(
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
valid_rows=valid_rows,
)
inputs, input_scale = self.quantize_input(
"w2",
inputs=buffers["activation_output"],
quanted_input=buffers.get("quanted_down_input", None),
)
self.humming_forward( self.humming_forward(
"w2", "w2",
@@ -743,12 +917,16 @@ class HummingIndexedExperts(HummingExpertsBase):
**moe_kwargs2, **moe_kwargs2,
) )
# expert_map masks any non-local id; num_valid_tokens bounds the
# persistent kernel to the real token rows [0, num_recv) so the padding
# tail is never iterated (CUDA-graph-safe device scalar).
moe_fused_mul_sum( moe_fused_mul_sum(
inputs=buffers["down_output"].view(*topk_ids.shape, -1), inputs=buffers["down_output"].view(*topk_ids.shape, -1),
topk_weights=topk_weights, topk_weights=topk_weights,
topk_ids=topk_ids, topk_ids=topk_ids,
expert_map=expert_map, expert_map=expert_map,
outputs=output, outputs=output,
num_valid_tokens=valid_tokens,
) )
@@ -787,10 +965,13 @@ class HummingGroupedExperts(HummingExpertsBase):
""" """
Standard apply implementation for Humming grouped experts. Standard apply implementation for Humming grouped experts.
Humming performs activation quantization internally and consumes the Note: Humming kernels handle weights internally through the layer
weights supplied by the modular kernel interface. object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
Results are written to the output buffer supplied by the modular path (Humming quantizes the w13 input itself); for block-FP8 activations
kernel, which may alias workspace13. it carries the scale computed before dispatch. It is permuted alongside
the tokens by moe_permute and forwarded to may_quant_input so Humming
skips the redundant w13 quantization. The output is written into
workspace13 via the buffer management.
""" """
assert not apply_router_weight_on_input assert not apply_router_weight_on_input
@@ -805,9 +986,9 @@ class HummingGroupedExperts(HummingExpertsBase):
) )
buffers["output"] = output buffers["output"] = output
hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute( hidden_states, a1q_scale, expert_first_token_offset, inv_perm, _ = moe_permute(
hidden_states=hidden_states, hidden_states=hidden_states,
a1q_scale=None, a1q_scale=a1q_scale,
topk_ids=topk_ids, topk_ids=topk_ids,
n_expert=global_num_experts, n_expert=global_num_experts,
n_local_expert=self.num_experts, n_local_expert=self.num_experts,
@@ -818,6 +999,7 @@ class HummingGroupedExperts(HummingExpertsBase):
inputs, input_scale = self.quantize_input( inputs, input_scale = self.quantize_input(
"w13", "w13",
inputs=hidden_states, inputs=hidden_states,
input_scale=a1q_scale,
quanted_input=buffers.get("quanted_gate_up_input", None), quanted_input=buffers.get("quanted_gate_up_input", None),
) )
@@ -901,15 +1083,21 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
""" """
Standard apply implementation for Humming batched grouped experts. Standard apply implementation for Humming batched grouped experts.
Humming performs activation quantization internally and consumes the Note: Humming kernels handle weights internally through the layer
weights supplied by the modular kernel interface. object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
Results are written to the output buffer supplied by the modular path (Humming quantizes the w13 input itself); for block-FP8 activations
kernel, which may alias workspace13. it carries the scale computed before dispatch, which is forwarded to
may_quant_input so Humming skips the redundant w13 quantization. The
output is written into workspace13 via the buffer management.
""" """
assert not apply_router_weight_on_input assert not apply_router_weight_on_input
assert expert_tokens_meta is not None assert expert_tokens_meta is not None
hidden_states = hidden_states.view(-1, hidden_states.size(-1)) hidden_states = hidden_states.view(-1, hidden_states.size(-1))
# Keep the (batched) block-FP8 scale row-aligned with the flattened
# [num_experts * max_tokens, K] hidden states above.
if a1q_scale is not None and a1q_scale.dim() == 3:
a1q_scale = a1q_scale.view(-1, a1q_scale.size(-1))
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids) valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
expert_num_tokens = expert_tokens_meta.expert_num_tokens expert_num_tokens = expert_tokens_meta.expert_num_tokens
@@ -925,6 +1113,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
inputs, input_scale = self.quantize_input( inputs, input_scale = self.quantize_input(
"w13", "w13",
inputs=hidden_states, inputs=hidden_states,
input_scale=a1q_scale,
quanted_input=buffers.get("quanted_gate_up_input", None), quanted_input=buffers.get("quanted_gate_up_input", None),
) )
@@ -100,8 +100,9 @@ class ExpertTokensMetadata:
Metadata regarding expert-token routing. Metadata regarding expert-token routing.
""" """
expert_num_tokens: torch.Tensor expert_num_tokens: torch.Tensor | None
expert_num_tokens_cpu: torch.Tensor | None expert_num_tokens_cpu: torch.Tensor | None
psum_recv_per_rank: torch.Tensor | None = None
@staticmethod @staticmethod
def make_from_list( def make_from_list(
@@ -901,6 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
*, *,
topk_ids: torch.Tensor | None = None, topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None, expert_map: torch.Tensor | None = None,
valid_rows: torch.Tensor | None = None,
) -> None: ) -> None:
apply_moe_activation( apply_moe_activation(
activation, activation,
@@ -909,6 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
activation_config=self.activation_config, activation_config=self.activation_config,
topk_ids=topk_ids, topk_ids=topk_ids,
expert_map=expert_map, expert_map=expert_map,
valid_rows=valid_rows,
) )
@abstractmethod @abstractmethod
@@ -3,7 +3,6 @@
import torch import torch
from torch._subclasses.fake_tensor import FakeTensor from torch._subclasses.fake_tensor import FakeTensor
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton from vllm.triton_utils import tl, triton
@@ -14,122 +13,75 @@ def moe_fused_mul_sum_kernel(
outputs_ptr, outputs_ptr,
top_ids_ptr, top_ids_ptr,
expert_map_ptr, expert_map_ptr,
num_tokens, num_valid_tokens_ptr,
stride_m, stride_m,
has_topk_ids: tl.constexpr,
has_expert_map: tl.constexpr, has_expert_map: tl.constexpr,
has_num_valid: tl.constexpr,
top_k: tl.constexpr, top_k: tl.constexpr,
size: tl.constexpr, hidden_size: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr, BLOCK_K: tl.constexpr,
): ):
pid_k = tl.program_id(0) # One CTA owns one token's output row. Resolve which slots to sum once per
pid_m = tl.program_id(1) # row (the decision is invariant over hidden tiles); the `if take` branch is
# block-uniform, so padding slots are skipped rather than masked-and-loaded.
pid_m = tl.program_id(0)
# Bound to the real recv rows [0, num_recv). Under cudagraph decode the grid
# is the static padded token count, and padding rows past num_recv carry
# stale ids and stale NaN in `inputs`; returning before any load leaves their
# output rows untouched instead of summing that garbage. The constexpr guard
# short-circuits, so num_valid_tokens_ptr is only read when it is provided.
if has_num_valid and pid_m >= tl.load(num_valid_tokens_ptr).to(tl.int32):
return
w_row = topk_weights_ptr + pid_m * top_k
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) takes = ()
offs_k = pid_k * BLOCK_K + tl.arange(0, BLOCK_K) weights = ()
if has_topk_ids:
any_valid = 0
for n in tl.static_range(top_k):
id_val = tl.load(top_ids_ptr + pid_m * top_k + n)
valid = id_val >= 0
any_valid += valid.to(tl.int32)
take = valid
if has_expert_map:
local_id = tl.load(expert_map_ptr + tl.where(valid, id_val, 0))
take = valid & (local_id >= 0)
takes += (take,) # type: ignore[assignment]
weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment]
# All-padding rows early-return untouched (decode contract past num_recv).
if any_valid == 0:
return
else:
for n in tl.static_range(top_k):
takes += (True,) # type: ignore[assignment]
weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment]
m_mask = offs_m < num_tokens n_tiles: tl.constexpr = (hidden_size + BLOCK_K - 1) // BLOCK_K
k_mask = offs_k < size a_row = inputs_ptr + pid_m * stride_m
mask = m_mask[:, None] & k_mask[None, :] out_row = outputs_ptr + pid_m * hidden_size
a_base = inputs_ptr + (offs_m * stride_m)[:, None] + offs_k[None, :] for t in tl.range(0, n_tiles):
b_base = topk_weights_ptr + offs_m * top_k offs_k = t * BLOCK_K + tl.arange(0, BLOCK_K)
kmask = offs_k < hidden_size
acc = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32) acc = tl.zeros((BLOCK_K,), dtype=tl.float32)
for n in tl.static_range(top_k):
for n in tl.static_range(top_k): if takes[n]:
b_val = tl.load(b_base + n, mask=m_mask, other=0.0).to(tl.float32) a = tl.load(a_row + n * hidden_size + offs_k, mask=kmask, other=0.0)
if has_expert_map: acc += a.to(tl.float32) * weights[n]
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0) tl.store(out_row + offs_k, acc.to(outputs_ptr.dtype.element_ty), mask=kmask)
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
a_vec = tl.load(
a_base + n * size,
mask=mask & expert_mask[:, None],
other=0.0,
).to(tl.float32)
else:
a_vec = tl.load(
a_base + n * size,
mask=mask,
other=0.0,
).to(tl.float32)
acc += a_vec * b_val[:, None]
out_ptrs = outputs_ptr + (offs_m * size)[:, None] + offs_k[None, :]
tl.store(
out_ptrs,
acc.to(outputs_ptr.dtype.element_ty),
mask=mask,
)
def _heuristic_config( def _heuristic_config(
num_tokens: int, hidden_size: int,
top_k: int,
size: int,
element_size: int, element_size: int,
): ):
is_fp32 = element_size > 2 is_fp32 = element_size > 2
is_sm90_plus = current_platform.has_device_capability(90) max_block_k = 256 if is_fp32 else 512
is_sm80_before = not current_platform.has_device_capability(80) BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
num_warps = 4 if is_fp32 else 2
if current_platform.has_device_capability(90): num_stages = 3
# SM90/SM100+: prefer small tiles + many CTAs. return BLOCK_K, num_warps, num_stages
if is_fp32:
BLOCK_M = 1 if num_tokens <= 4 else 2
else:
if num_tokens <= 4:
BLOCK_M = 1
elif num_tokens <= 128:
BLOCK_M = 2
else:
BLOCK_M = 4
elif is_fp32:
if num_tokens <= 4:
BLOCK_M = 1
elif num_tokens <= 32:
BLOCK_M = 2
elif num_tokens <= 128:
BLOCK_M = 4
else:
BLOCK_M = 4
else:
if num_tokens <= 4:
BLOCK_M = 1
elif num_tokens <= 32:
BLOCK_M = 2
elif num_tokens <= 128:
BLOCK_M = 4
elif num_tokens <= 1024:
BLOCK_M = 16
else:
BLOCK_M = 8
if is_fp32:
max_block_k = 256
elif is_sm80_before or is_sm90_plus:
max_block_k = 512
else:
max_block_k = 1024
BLOCK_K = min(triton.next_power_of_2(size), max_block_k)
BLOCK_K = max(BLOCK_K, 256)
total = BLOCK_M * BLOCK_K
if is_fp32:
num_warps = max(8, min(16, total // 64))
else:
num_warps = max(4, min(16, total // 256))
if is_sm80_before:
num_warps = min(num_warps, 8)
num_stages = 2
elif is_sm90_plus:
num_warps = min(num_warps, 8)
num_stages = 4 if total <= 2048 else 2
else:
num_stages = 4 if total <= 2048 else 2
return BLOCK_M, BLOCK_K, num_warps, num_stages
def moe_fused_mul_sum( def moe_fused_mul_sum(
@@ -138,6 +90,7 @@ def moe_fused_mul_sum(
outputs: torch.Tensor | None = None, outputs: torch.Tensor | None = None,
topk_ids: torch.Tensor | None = None, topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None, expert_map: torch.Tensor | None = None,
num_valid_tokens: torch.Tensor | None = None,
) -> torch.Tensor: ) -> torch.Tensor:
""" """
Fused kernel for MoE (Mixture of Experts) to perform weighted summation Fused kernel for MoE (Mixture of Experts) to perform weighted summation
@@ -150,10 +103,21 @@ def moe_fused_mul_sum(
Shape: (num_tokens, top_k). Shape: (num_tokens, top_k).
outputs: Optional pre-allocated output tensor. outputs: Optional pre-allocated output tensor.
Shape: (num_tokens, hidden_size). Shape: (num_tokens, hidden_size).
topk_ids: Optional indices of the top-k experts. Used when topk_ids: Optional indices of the top-k experts. Shape:
`expert_map` is provided. Shape: (num_tokens, top_k). (num_tokens, top_k). A value of -1 marks a slot the expert GEMM
skipped; those slots are excluded from the sum. When provided, rows
with all top ids < 0 (worst-case padding) are skipped and their
output rows left untouched. Required when `expert_map` is provided.
expert_map: Optional mapping for Expert Parallelism. A value < 0 expert_map: Optional mapping for Expert Parallelism. A value < 0
indicates an invalid token/expert pair that will be skipped. indicates an invalid token/expert pair that will be skipped. Only
needed when `topk_ids` may contain non-local expert ids; if every
non-(-1) id is already a local expert, leave it None to skip the
redundant per-slot lookup.
num_valid_tokens: Optional device scalar (1-element tensor) holding the
number of real token rows (num_recv for a decode dispatch). When
provided, rows past it are left untouched, so the static cudagraph
grid never sums stale padding rows. Pass the token count, not
token*top_k.
Returns: Returns:
The fused weighted sum of expert outputs. The fused weighted sum of expert outputs.
@@ -166,34 +130,40 @@ def moe_fused_mul_sum(
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16) assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16) assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
num_tokens, top_k, size = inputs.shape num_tokens, top_k, hidden_size = inputs.shape
output_shape = (num_tokens, size) output_shape = (num_tokens, hidden_size)
if outputs is None: if outputs is None:
outputs = torch.empty(output_shape, dtype=inputs.dtype, device=inputs.device) outputs = torch.empty(output_shape, dtype=inputs.dtype, device=inputs.device)
assert outputs.shape == output_shape assert outputs.shape == output_shape
assert topk_weights.shape == (num_tokens, top_k) assert topk_weights.shape == (num_tokens, top_k)
assert expert_map is None or topk_ids is not None, (
"topk_ids is required to interpret expert_map"
)
if topk_ids is not None:
assert topk_ids.shape == (num_tokens, top_k)
assert topk_ids.is_contiguous()
assert topk_ids.dtype in (torch.int32, torch.int64)
if not isinstance(inputs, FakeTensor): if not isinstance(inputs, FakeTensor):
BLOCK_M, BLOCK_K, num_warps, num_stages = _heuristic_config( BLOCK_K, num_warps, num_stages = _heuristic_config(
num_tokens, hidden_size,
top_k,
size,
inputs.element_size(), inputs.element_size(),
) )
grid = (triton.cdiv(size, BLOCK_K), triton.cdiv(num_tokens, BLOCK_M)) grid = (num_tokens,)
moe_fused_mul_sum_kernel[grid]( moe_fused_mul_sum_kernel[grid](
inputs, inputs,
topk_weights, topk_weights,
outputs, outputs,
topk_ids, topk_ids,
expert_map, expert_map,
num_tokens, num_valid_tokens,
top_k * size, top_k * hidden_size,
topk_ids is not None,
expert_map is not None, expert_map is not None,
num_valid_tokens is not None,
top_k, top_k,
size, hidden_size,
BLOCK_M,
BLOCK_K, BLOCK_K,
num_warps=num_warps, num_warps=num_warps,
num_stages=num_stages, num_stages=num_stages,
@@ -32,7 +32,7 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
- Worst-case tensor allocation; padding rows zeroed via - Worst-case tensor allocation; padding rows zeroed via
handle.psum_num_recv_tokens_per_scaleup_rank handle.psum_num_recv_tokens_per_scaleup_rank
- Fully cudagraph-capturable - Fully cudagraph-capturable
- Expert kernel sorts internally (expert_tokens_meta=None) - Expert kernel sorts internally (expert_tokens_meta carries no counts)
**Prefill mode (use_cudagraph=False):** **Prefill mode (use_cudagraph=False):**
- do_expand=True, do_cpu_sync=True - do_expand=True, do_cpu_sync=True
@@ -266,6 +266,18 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
if recv_topk_weights is not None and recv_topk_weights.ndim == 1: if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
recv_topk_weights = recv_topk_weights.unsqueeze(1) recv_topk_weights = recv_topk_weights.unsqueeze(1)
if self.use_cudagraph:
# Carry the per-rank prefix sum so SiTU can skip padding rows.
# expert_num_tokens stays None: count-based consumers (DeepGEMM,
# Triton) must treat a None field as "no counts" and derive their
# own, exactly as in the meta-absent decode case.
if expert_tokens_meta is None:
expert_tokens_meta = mk.ExpertTokensMetadata(
expert_num_tokens=None,
expert_num_tokens_cpu=None,
)
expert_tokens_meta.psum_recv_per_rank = psum_recv_per_rank
if not quant_config.is_block_quantized and not defer_input_quant: if not quant_config.is_block_quantized and not defer_input_quant:
expert_x_scale = None expert_x_scale = None
if expert_x.numel() != 0: if expert_x.numel() != 0:
@@ -307,7 +307,18 @@ def _humming_input_schema_to_quant_key(
gs = schema.input_scale_group_size gs = schema.input_scale_group_size
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32 # Pick the scale dtype the Humming kernel actually consumes. An explicit
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
# float32 scale. Getting this right lets a grouped FP8 activation match
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
if schema.input_scale_dtype is not None:
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
elif gs == 32:
scale_dtype = MXFP_SCALE_DTYPE
else:
scale_dtype = torch.float32
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape) scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
@@ -553,6 +564,7 @@ def make_humming_moe_quant_config(
quant_dtype: torch.dtype | str | None, quant_dtype: torch.dtype | str | None,
weight_dtype: torch.dtype | str | None, weight_dtype: torch.dtype | str | None,
weight_group_shape: GroupShape | None = None, weight_group_shape: GroupShape | None = None,
activation_group_shape: GroupShape | None = None,
w1_scale: torch.Tensor | None = None, w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None, w2_scale: torch.Tensor | None = None,
w1_zp: torch.Tensor | None = None, w1_zp: torch.Tensor | None = None,
@@ -569,7 +581,14 @@ def make_humming_moe_quant_config(
assert humming_configs is not None assert humming_configs is not None
if quant_dtype is None: if quant_dtype is None:
a_quant_desc = FusedMoEQuantDesc(dtype=None) a_quant_desc = FusedMoEQuantDesc(dtype=None)
elif activation_group_shape is not None:
# Pre-dispatch quantization.
a_quant_desc = FusedMoEQuantDesc(
dtype=quant_dtype, shape=activation_group_shape
)
else: else:
# Deferred path: Humming quantizes the activation internally, so the
# descriptor only needs a non-None dtype to mark it as quantized.
shape = GroupShape(row=1, col=-1) shape = GroupShape(row=1, col=-1)
a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape) a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape)
@@ -621,6 +640,19 @@ def get_humming_moe_quant_config(
else: else:
q_dtype = str(input_schema.a_dtype) q_dtype = str(input_schema.a_dtype)
# Block-FP8 (group-128) activations are quantized *before* the EP all-to-all
# dispatch (so FP8 rather than BF16 crosses the interconnect) and consumed by
# Humming as-is.
activation_group_shape: GroupShape | None = None
input_scale_group_size = getattr(input_schema, "input_scale_group_size", 0) or 0
if (
q_dtype is not None
and q_dtype.startswith("float8")
and input_scale_group_size == 128
):
q_dtype = _HUMMING_TO_QUANT_DTYPE.get(input_schema.a_dtype, FP8_DTYPE)
activation_group_shape = GroupShape(row=1, col=input_scale_group_size)
weight_scale_group_size = weight_schema.weight_scale_group_size weight_scale_group_size = weight_schema.weight_scale_group_size
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
weight_group_shape: tuple[int, ...] = () weight_group_shape: tuple[int, ...] = ()
@@ -638,6 +670,7 @@ def get_humming_moe_quant_config(
quant_dtype=q_dtype, quant_dtype=q_dtype,
weight_dtype=str(weight_schema.b_dtype), weight_dtype=str(weight_schema.b_dtype),
weight_group_shape=weight_group_shape, weight_group_shape=weight_group_shape,
activation_group_shape=activation_group_shape,
w1_scale=getattr(layer, "w13_weight_scale", None), w1_scale=getattr(layer, "w13_weight_scale", None),
w1_gscale=getattr(layer, "w13_weight_scale_2", None), w1_gscale=getattr(layer, "w13_weight_scale_2", None),
w1_zp=getattr(layer, "w13_zero_point", None), w1_zp=getattr(layer, "w13_zero_point", None),