[Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Elvir Crnčević <elvircrn@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Tyler Michael Smith
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9c8e90eb26
commit
ca29cd48ba
@@ -1,9 +1,17 @@
|
|||||||
#include <cuda.h>
|
#include <cuda.h>
|
||||||
|
#include <cuda_runtime.h>
|
||||||
#include <torch/csrc/stable/tensor.h>
|
#include <torch/csrc/stable/tensor.h>
|
||||||
|
#include <torch/headeronly/util/Float8_e4m3fn.h>
|
||||||
|
|
||||||
|
#ifndef USE_ROCM
|
||||||
|
#include <cuda_fp8.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
|
#include <type_traits>
|
||||||
|
|
||||||
#include "../cuda_compat.h"
|
#include "../cuda_compat.h"
|
||||||
|
#include "async_util.cuh"
|
||||||
#include "cuda_vec_utils.cuh"
|
#include "cuda_vec_utils.cuh"
|
||||||
#include "dispatch_utils.h"
|
#include "dispatch_utils.h"
|
||||||
#include "torch_utils.h"
|
#include "torch_utils.h"
|
||||||
@@ -471,15 +479,31 @@ __global__ void swigluoai_and_mul_kernel(
|
|||||||
// gate_out = beta * tanh(gate / beta) * sigmoid(gate)
|
// gate_out = beta * tanh(gate / beta) * sigmoid(gate)
|
||||||
// up_out = (linear_beta > 0) ? linear_beta * tanh(up / linear_beta) : up
|
// up_out = (linear_beta > 0) ? linear_beta * tanh(up / linear_beta) : up
|
||||||
// out = gate_out * up_out
|
// out = gate_out * up_out
|
||||||
// Compute is done in fp32 and written straight to `out` -- no intermediate
|
__device__ __forceinline__ float situ_tanh(float x) { return tanhf(x); }
|
||||||
// tensors and no full-tensor fp32 upcast (the pure-torch forward_native
|
|
||||||
// allocated ~8 fp32 temporaries per call, which blows up MoE profiling).
|
// Kimi-K3 SITU params; baked into the fused LDG kernel to fold at compile time.
|
||||||
|
static constexpr float SITU_BETA = 4.0f;
|
||||||
|
static constexpr float SITU_LINEAR_BETA = 25.0f;
|
||||||
|
__device__ __forceinline__ float situ_activation(float g, float u, float beta,
|
||||||
|
float linear_beta,
|
||||||
|
bool clamp_up, float inv_beta,
|
||||||
|
float inv_linear_beta) {
|
||||||
|
// sigmoid(g) == (1 + tanh(g/2)) / 2.
|
||||||
|
const float gate_out =
|
||||||
|
(0.5f * beta) * situ_tanh(g * inv_beta) * (1.0f + situ_tanh(g * 0.5f));
|
||||||
|
const float up_out =
|
||||||
|
clamp_up ? linear_beta * situ_tanh(u * inv_linear_beta) : u;
|
||||||
|
return gate_out * up_out;
|
||||||
|
}
|
||||||
|
|
||||||
template <typename scalar_t>
|
template <typename scalar_t>
|
||||||
__global__ void situ_and_mul_kernel(
|
__global__ void situ_and_mul_kernel(
|
||||||
scalar_t* __restrict__ out, // [..., d]
|
scalar_t* __restrict__ out, // [..., d]
|
||||||
const scalar_t* __restrict__ input, // [..., 2, d]
|
const scalar_t* __restrict__ input, // [..., 2, d]
|
||||||
const int d, const float beta, const float linear_beta) {
|
const int d, const float beta, const float linear_beta,
|
||||||
|
const int64_t* __restrict__ valid_rows_ptr) {
|
||||||
const int64_t row = blockIdx.x;
|
const int64_t row = blockIdx.x;
|
||||||
|
if (valid_rows_ptr != nullptr && row >= *valid_rows_ptr) return;
|
||||||
const scalar_t* gate_ptr = input + row * 2 * d;
|
const scalar_t* gate_ptr = input + row * 2 * d;
|
||||||
const scalar_t* up_ptr = gate_ptr + d;
|
const scalar_t* up_ptr = gate_ptr + d;
|
||||||
scalar_t* out_ptr = out + row * d;
|
scalar_t* out_ptr = out + row * d;
|
||||||
@@ -489,10 +513,258 @@ __global__ void situ_and_mul_kernel(
|
|||||||
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
|
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
|
||||||
const float g = (float)VLLM_LDG(&gate_ptr[idx]);
|
const float g = (float)VLLM_LDG(&gate_ptr[idx]);
|
||||||
const float u = (float)VLLM_LDG(&up_ptr[idx]);
|
const float u = (float)VLLM_LDG(&up_ptr[idx]);
|
||||||
const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g));
|
out_ptr[idx] = (scalar_t)situ_activation(g, u, beta, linear_beta, clamp_up,
|
||||||
const float up_out =
|
inv_beta, inv_linear_beta);
|
||||||
clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u;
|
}
|
||||||
out_ptr[idx] = (scalar_t)(gate_out * up_out);
|
}
|
||||||
|
|
||||||
|
// Match Humming's hardware FP8 conversion. c10::Float8_e4m3fn's software cast
|
||||||
|
// can round ties differently from `cvt.rn.satfinite.e4m3.f32`.
|
||||||
|
__device__ __forceinline__ c10::Float8_e4m3fn cvt_fp8_hw(float x) {
|
||||||
|
#if !defined(USE_ROCM) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
|
||||||
|
return c10::Float8_e4m3fn(__nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3),
|
||||||
|
c10::Float8_e4m3fn::from_bits());
|
||||||
|
#else
|
||||||
|
return static_cast<c10::Float8_e4m3fn>(x);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename fp8_type>
|
||||||
|
__device__ __forceinline__ fp8_type quant_to_fp8(float val, float inv_scale,
|
||||||
|
float fp8_max) {
|
||||||
|
float x = val * inv_scale;
|
||||||
|
x = fmaxf(-fp8_max, fminf(x, fp8_max));
|
||||||
|
if constexpr (std::is_same_v<fp8_type, c10::Float8_e4m3fn>) {
|
||||||
|
return cvt_fp8_hw(x);
|
||||||
|
} else {
|
||||||
|
return static_cast<fp8_type>(x);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
__device__ __forceinline__ float warp_reduce_max(float v) {
|
||||||
|
#pragma unroll
|
||||||
|
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) {
|
||||||
|
v = fmaxf(v, VLLM_SHFL_XOR_SYNC(v, offset));
|
||||||
|
}
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
__device__ __forceinline__ float warp_reduce_absmax(float v) {
|
||||||
|
return warp_reduce_max(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename scalar_t, typename fp8_type, int THREADS, int NUM_STAGES,
|
||||||
|
int GROUP_SIZE, int GRID_DIM, int D>
|
||||||
|
__global__ void situ_and_mul_quant_group_pipelined_kernel(
|
||||||
|
fp8_type* __restrict__ out, float* __restrict__ scale_out,
|
||||||
|
const scalar_t* __restrict__ input, const int64_t num_rows,
|
||||||
|
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
|
||||||
|
const int tid = threadIdx.x;
|
||||||
|
const int warp_id = tid / WARP_SIZE;
|
||||||
|
const int lane_id = tid % WARP_SIZE;
|
||||||
|
static constexpr int NUM_WARPS = THREADS / WARP_SIZE;
|
||||||
|
static constexpr int NUM_GROUPS = D / GROUP_SIZE;
|
||||||
|
const int64_t row_bound =
|
||||||
|
valid_rows_ptr != nullptr
|
||||||
|
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
|
||||||
|
: num_rows;
|
||||||
|
|
||||||
|
static_assert(NUM_GROUPS % 4 == 0,
|
||||||
|
"float4 scale fill needs NUM_GROUPS % 4 == 0");
|
||||||
|
static constexpr int NG4 = NUM_GROUPS / 4;
|
||||||
|
const int64_t pad_start4 = row_bound * NG4;
|
||||||
|
const int64_t pad_end4 = num_rows * NG4;
|
||||||
|
float4* scale4 = reinterpret_cast<float4*>(scale_out);
|
||||||
|
constexpr float4 ones{1.0f, 1.0f, 1.0f, 1.0f};
|
||||||
|
for (int64_t i = pad_start4 + (int64_t)blockIdx.x * THREADS + tid;
|
||||||
|
i < pad_end4; i += (int64_t)GRID_DIM * THREADS) {
|
||||||
|
scale4[i] = ones;
|
||||||
|
}
|
||||||
|
|
||||||
|
if constexpr (sizeof(scalar_t) == 2) {
|
||||||
|
static constexpr int LD_ELTS = 16 / sizeof(scalar_t);
|
||||||
|
static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE;
|
||||||
|
static constexpr int STAGE_ELTS = 2 * GROUP_SIZE;
|
||||||
|
|
||||||
|
extern __shared__ __align__(16) unsigned char smem_raw[];
|
||||||
|
scalar_t* warp_smem = reinterpret_cast<scalar_t*>(smem_raw) +
|
||||||
|
(size_t)warp_id * NUM_STAGES * STAGE_ELTS;
|
||||||
|
|
||||||
|
static constexpr float beta = SITU_BETA;
|
||||||
|
static constexpr float linear_beta = SITU_LINEAR_BETA;
|
||||||
|
static constexpr bool clamp_up = linear_beta > 0.0f;
|
||||||
|
static constexpr float inv_beta = 1.0f / beta;
|
||||||
|
static constexpr float inv_linear_beta =
|
||||||
|
clamp_up ? 1.0f / linear_beta : 0.0f;
|
||||||
|
static constexpr float FP8_MAX =
|
||||||
|
std::is_same_v<fp8_type, c10::Float8_e4m3fn> ? 448.0f : 224.0f;
|
||||||
|
static constexpr float inv_fp8_max = 1.0f / FP8_MAX;
|
||||||
|
|
||||||
|
static_assert(
|
||||||
|
NUM_GROUPS % NUM_WARPS == 0,
|
||||||
|
"constexpr num_iters requires groups evenly split across warps");
|
||||||
|
static constexpr int num_iters = NUM_GROUPS / NUM_WARPS;
|
||||||
|
|
||||||
|
const bool up_half = lane_id >= WARP_SIZE / 2;
|
||||||
|
const int lane_l = up_half ? lane_id - WARP_SIZE / 2 : lane_id;
|
||||||
|
const int lane_src_off = (up_half ? D : 0) + lane_l * LD_ELTS;
|
||||||
|
const int lane_dst_off = (up_half ? GROUP_SIZE : 0) + lane_l * LD_ELTS;
|
||||||
|
static constexpr int warp_stride = NUM_WARPS * GROUP_SIZE;
|
||||||
|
|
||||||
|
const scalar_t* src_ptr = input + warp_id * GROUP_SIZE + lane_src_off;
|
||||||
|
static constexpr int src_row_stride = GRID_DIM * 2 * D;
|
||||||
|
const scalar_t* row_src = src_ptr + blockIdx.x * 2 * D;
|
||||||
|
|
||||||
|
uint32_t* out_ptr =
|
||||||
|
reinterpret_cast<uint32_t*>(out + warp_id * GROUP_SIZE) + lane_id;
|
||||||
|
static constexpr int out_row_stride = GRID_DIM * D / 4;
|
||||||
|
uint32_t* row_out = out_ptr + blockIdx.x * (D / 4);
|
||||||
|
float* scale_ptr = scale_out + warp_id;
|
||||||
|
static constexpr int scale_row_stride = GRID_DIM * NUM_GROUPS;
|
||||||
|
float* row_scale = scale_ptr + blockIdx.x * NUM_GROUPS;
|
||||||
|
|
||||||
|
#pragma unroll 1
|
||||||
|
for (int64_t row = blockIdx.x; row < row_bound; row += GRID_DIM,
|
||||||
|
row_src += src_row_stride, row_out += out_row_stride,
|
||||||
|
row_scale += scale_row_stride) {
|
||||||
|
const scalar_t* src = row_src;
|
||||||
|
|
||||||
|
auto issue_load = [&](int it, int slot) {
|
||||||
|
if (it < num_iters) {
|
||||||
|
cuda_async::cp_async_shared_global_16_cg(
|
||||||
|
warp_smem + (size_t)slot * STAGE_ELTS + lane_dst_off, src);
|
||||||
|
src += warp_stride;
|
||||||
|
}
|
||||||
|
cuda_async::cp_async_commit_group();
|
||||||
|
};
|
||||||
|
|
||||||
|
int load_slot = 0;
|
||||||
|
auto bump = [](int s) { return s + 1 == NUM_STAGES ? 0 : s + 1; };
|
||||||
|
#pragma unroll
|
||||||
|
for (int s = 0; s < NUM_STAGES - 1; s++) {
|
||||||
|
issue_load(s, load_slot);
|
||||||
|
load_slot = bump(load_slot);
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t* out_st = row_out;
|
||||||
|
float* scale_st = row_scale;
|
||||||
|
|
||||||
|
int comp_slot = 0;
|
||||||
|
for (int it = 0; it < num_iters; it++) {
|
||||||
|
issue_load(it + NUM_STAGES - 1, load_slot);
|
||||||
|
load_slot = bump(load_slot);
|
||||||
|
cuda_async::cp_async_wait_group<NUM_STAGES - 1>();
|
||||||
|
|
||||||
|
const scalar_t* stage = warp_smem + (size_t)comp_slot * STAGE_ELTS;
|
||||||
|
comp_slot = bump(comp_slot);
|
||||||
|
static_assert(ELTS_PER_LANE == 4,
|
||||||
|
"expects GROUP_SIZE == 4 * WARP_SIZE");
|
||||||
|
union V {
|
||||||
|
float2 f2;
|
||||||
|
scalar_t s[ELTS_PER_LANE];
|
||||||
|
};
|
||||||
|
const float2* gate2 = reinterpret_cast<const float2*>(stage);
|
||||||
|
const float2* up2 = reinterpret_cast<const float2*>(stage + GROUP_SIZE);
|
||||||
|
const V gv{gate2[lane_id]};
|
||||||
|
const V uv{up2[lane_id]};
|
||||||
|
const scalar_t* gs = gv.s;
|
||||||
|
const scalar_t* us = uv.s;
|
||||||
|
|
||||||
|
float acts[ELTS_PER_LANE];
|
||||||
|
float thread_max = 0.0f;
|
||||||
|
#pragma unroll
|
||||||
|
for (int e = 0; e < ELTS_PER_LANE; e++) {
|
||||||
|
acts[e] = (float)(scalar_t)situ_activation(
|
||||||
|
(float)gs[e], (float)us[e], beta, linear_beta, clamp_up, inv_beta,
|
||||||
|
inv_linear_beta);
|
||||||
|
thread_max = fmaxf(thread_max, fabsf(acts[e]));
|
||||||
|
}
|
||||||
|
const float absmax = fmaxf(warp_reduce_absmax(thread_max), 1e-30f);
|
||||||
|
const float scale = absmax * inv_fp8_max;
|
||||||
|
if (lane_id == 0) *scale_st = scale;
|
||||||
|
scale_st += NUM_WARPS;
|
||||||
|
const float inv_scale = __fdividef(1.0f, scale);
|
||||||
|
|
||||||
|
union O {
|
||||||
|
uint32_t u;
|
||||||
|
fp8_type f[ELTS_PER_LANE];
|
||||||
|
};
|
||||||
|
O o;
|
||||||
|
#pragma unroll
|
||||||
|
for (int e = 0; e < ELTS_PER_LANE; e++) {
|
||||||
|
o.f[e] = quant_to_fp8<fp8_type>(acts[e], inv_scale, FP8_MAX);
|
||||||
|
}
|
||||||
|
out_st[0] = o.u;
|
||||||
|
out_st += NUM_WARPS * GROUP_SIZE / 4;
|
||||||
|
}
|
||||||
|
cuda_async::cp_async_wait_group<0>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scalar fallback for the group path (odd d or the fp32 dispatch branch).
|
||||||
|
template <typename scalar_t, typename fp8_type, int GROUP_SIZE>
|
||||||
|
__global__ void situ_and_mul_quant_group_scalar_kernel(
|
||||||
|
fp8_type* __restrict__ out, // [num_tokens, d]
|
||||||
|
float* __restrict__ scale_out, // [num_tokens, num_groups]
|
||||||
|
const scalar_t* __restrict__ input, // [num_tokens, 2, d]
|
||||||
|
const int d, const int num_groups, const float beta,
|
||||||
|
const float linear_beta, const int64_t num_rows,
|
||||||
|
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
|
||||||
|
static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE;
|
||||||
|
const int tid = threadIdx.x;
|
||||||
|
const int warp_id = tid / WARP_SIZE;
|
||||||
|
const int lane_id = tid % WARP_SIZE;
|
||||||
|
const int num_warps = blockDim.x / WARP_SIZE;
|
||||||
|
const int64_t row_bound =
|
||||||
|
valid_rows_ptr != nullptr
|
||||||
|
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
|
||||||
|
: num_rows;
|
||||||
|
|
||||||
|
const bool clamp_up = linear_beta > 0.0f;
|
||||||
|
const float inv_beta = __fdividef(1.0f, beta);
|
||||||
|
const float inv_linear_beta = clamp_up ? __fdividef(1.0f, linear_beta) : 0.0f;
|
||||||
|
static constexpr float FP8_MAX =
|
||||||
|
std::is_same_v<fp8_type, c10::Float8_e4m3fn> ? 448.0f : 224.0f;
|
||||||
|
static constexpr float inv_fp8_max = 1.0f / FP8_MAX;
|
||||||
|
|
||||||
|
for (int64_t row = blockIdx.x; row < row_bound; row += gridDim.x) {
|
||||||
|
const scalar_t* gate_ptr = input + row * 2 * (int64_t)d;
|
||||||
|
const scalar_t* up_ptr = gate_ptr + d;
|
||||||
|
fp8_type* out_ptr = out + row * (int64_t)d;
|
||||||
|
float* scale_row = scale_out + row * (int64_t)num_groups;
|
||||||
|
|
||||||
|
for (int g = warp_id; g < num_groups; g += num_warps) {
|
||||||
|
const int base = g * GROUP_SIZE;
|
||||||
|
float acts[ELTS_PER_LANE];
|
||||||
|
float thread_max = 0.0f;
|
||||||
|
#pragma unroll
|
||||||
|
for (int e = 0; e < ELTS_PER_LANE; e++) {
|
||||||
|
const int idx = base + e * WARP_SIZE + lane_id;
|
||||||
|
const float gv = (float)VLLM_LDG(&gate_ptr[idx]);
|
||||||
|
const float uv = (float)VLLM_LDG(&up_ptr[idx]);
|
||||||
|
acts[e] = (float)(scalar_t)situ_activation(
|
||||||
|
gv, uv, beta, linear_beta, clamp_up, inv_beta, inv_linear_beta);
|
||||||
|
thread_max = fmaxf(thread_max, fabsf(acts[e]));
|
||||||
|
}
|
||||||
|
const float absmax = fmaxf(warp_reduce_max(thread_max), 1e-30f);
|
||||||
|
const float scale = absmax * inv_fp8_max;
|
||||||
|
if (lane_id == 0) scale_row[g] = scale;
|
||||||
|
const float inv_scale = __fdividef(1.0f, scale);
|
||||||
|
#pragma unroll
|
||||||
|
for (int e = 0; e < ELTS_PER_LANE; e++) {
|
||||||
|
const int idx = base + e * WARP_SIZE + lane_id;
|
||||||
|
out_ptr[idx] = quant_to_fp8<fp8_type>(acts[e], inv_scale, FP8_MAX);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fill skipped padding-row scales with 1 so they don't feed NaN into w2.
|
||||||
|
const int64_t pad_start = row_bound * (int64_t)num_groups;
|
||||||
|
const int64_t pad_end = num_rows * (int64_t)num_groups;
|
||||||
|
for (int64_t i = pad_start + (int64_t)blockIdx.x * blockDim.x + tid;
|
||||||
|
i < pad_end; i += (int64_t)gridDim.x * blockDim.x) {
|
||||||
|
scale_out[i] = 1.0f;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -519,10 +791,8 @@ __global__ void masked_situ_and_mul_kernel(
|
|||||||
scalar_t* out_ptr = out + row * d;
|
scalar_t* out_ptr = out + row * d;
|
||||||
const float g = (float)VLLM_LDG(&gate_ptr[idx]);
|
const float g = (float)VLLM_LDG(&gate_ptr[idx]);
|
||||||
const float u = (float)VLLM_LDG(&up_ptr[idx]);
|
const float u = (float)VLLM_LDG(&up_ptr[idx]);
|
||||||
const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g));
|
out_ptr[idx] = (scalar_t)situ_activation(g, u, beta, linear_beta, clamp_up,
|
||||||
const float up_out =
|
inv_beta, inv_linear_beta);
|
||||||
clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u;
|
|
||||||
out_ptr[idx] = (scalar_t)(gate_out * up_out);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -620,7 +890,8 @@ void swigluoai_and_mul(torch::stable::Tensor& out, // [..., d]
|
|||||||
// through), matching SituAndMul(linear_beta=None) on the Python side.
|
// through), matching SituAndMul(linear_beta=None) on the Python side.
|
||||||
void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||||
torch::stable::Tensor& input, // [..., 2 * d]
|
torch::stable::Tensor& input, // [..., 2 * d]
|
||||||
double beta, double linear_beta) {
|
double beta, double linear_beta,
|
||||||
|
std::optional<torch::stable::Tensor> valid_rows) {
|
||||||
int d = input.size(-1) / 2;
|
int d = input.size(-1) / 2;
|
||||||
int64_t num_tokens = input.numel() / input.size(-1);
|
int64_t num_tokens = input.numel() / input.size(-1);
|
||||||
if (num_tokens == 0) {
|
if (num_tokens == 0) {
|
||||||
@@ -628,6 +899,14 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
|||||||
}
|
}
|
||||||
dim3 grid(num_tokens);
|
dim3 grid(num_tokens);
|
||||||
dim3 block(std::min(d, 1024));
|
dim3 block(std::min(d, 1024));
|
||||||
|
const int64_t* valid_rows_ptr = nullptr;
|
||||||
|
if (valid_rows.has_value()) {
|
||||||
|
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
|
||||||
|
input.get_device_index(),
|
||||||
|
"situ_and_mul: valid_rows must be a CUDA tensor on the "
|
||||||
|
"same device as input");
|
||||||
|
valid_rows_ptr = valid_rows->const_data_ptr<int64_t>();
|
||||||
|
}
|
||||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||||
input.get_device_index());
|
input.get_device_index());
|
||||||
const cudaStream_t stream = get_current_cuda_stream();
|
const cudaStream_t stream = get_current_cuda_stream();
|
||||||
@@ -635,7 +914,7 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
|||||||
input.scalar_type(), "situ_and_mul_kernel", [&] {
|
input.scalar_type(), "situ_and_mul_kernel", [&] {
|
||||||
vllm::situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
vllm::situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
||||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
|
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
|
||||||
d, (float)beta, (float)linear_beta);
|
d, (float)beta, (float)linear_beta, valid_rows_ptr);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -663,6 +942,114 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
|
|||||||
(float)beta, (float)linear_beta);
|
(float)beta, (float)linear_beta);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fused SITU activation and block-FP8 quantization for Humming w2.
|
||||||
|
// `valid_rows` is the int32 DeepEP token count (psum[-1]); rows = it * `topk`,
|
||||||
|
// so padding is excluded on-device and skipped rows receive scale 1.
|
||||||
|
void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
|
||||||
|
torch::stable::Tensor& scale, // [..., 1 or d/128]
|
||||||
|
torch::stable::Tensor& input, // [..., 2 * d]
|
||||||
|
double beta, double linear_beta, int64_t group_size,
|
||||||
|
std::optional<torch::stable::Tensor> valid_rows,
|
||||||
|
int64_t topk) {
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn ||
|
||||||
|
out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fnuz,
|
||||||
|
"situ_and_mul_quant output must be FP8 e4m3");
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
input.scalar_type() == torch::headeronly::ScalarType::Half ||
|
||||||
|
input.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||||
|
"situ_and_mul_quant input must be FP16 or BF16");
|
||||||
|
STD_TORCH_CHECK(scale.scalar_type() == torch::headeronly::ScalarType::Float,
|
||||||
|
"situ_and_mul_quant scale must be float32");
|
||||||
|
STD_TORCH_CHECK(input.is_cuda() && out.is_cuda() && scale.is_cuda() &&
|
||||||
|
out.get_device_index() == input.get_device_index() &&
|
||||||
|
scale.get_device_index() == input.get_device_index(),
|
||||||
|
"situ_and_mul_quant: input, out and scale must be CUDA "
|
||||||
|
"tensors on the same device");
|
||||||
|
int d = input.size(-1) / 2;
|
||||||
|
int64_t num_tokens = input.numel() / input.size(-1);
|
||||||
|
if (num_tokens == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
STD_TORCH_CHECK(out.size(-1) == d && out.numel() == num_tokens * (int64_t)d,
|
||||||
|
"situ_and_mul_quant: out shape must be [num_tokens, d]");
|
||||||
|
const int32_t* valid_rows_ptr = nullptr;
|
||||||
|
if (valid_rows.has_value()) {
|
||||||
|
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
|
||||||
|
input.get_device_index(),
|
||||||
|
"situ_and_mul_quant: valid_rows must be a CUDA tensor on "
|
||||||
|
"the same device as input");
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
valid_rows->scalar_type() == torch::headeronly::ScalarType::Int,
|
||||||
|
"situ_and_mul_quant: valid_rows must be int32 (psum count)");
|
||||||
|
valid_rows_ptr = valid_rows->const_data_ptr<int32_t>();
|
||||||
|
}
|
||||||
|
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||||
|
input.get_device_index());
|
||||||
|
const cudaStream_t stream = get_current_cuda_stream();
|
||||||
|
static constexpr int BLOCKS_PER_SM = 8; // matches kernel __launch_bounds__
|
||||||
|
static constexpr int SM_COUNT = 132; // H200 (GH100, 132 SMs)
|
||||||
|
static constexpr int GRID_DIM = SM_COUNT * BLOCKS_PER_SM;
|
||||||
|
|
||||||
|
STD_TORCH_CHECK(group_size == 128,
|
||||||
|
"situ_and_mul_quant: only group_size 128 (block-FP8) "
|
||||||
|
"supported, got ",
|
||||||
|
group_size);
|
||||||
|
STD_TORCH_CHECK(d % group_size == 0, "situ_and_mul_quant: d (", d,
|
||||||
|
") must be divisible by group_size ", group_size);
|
||||||
|
{
|
||||||
|
const int num_groups = d / (int)group_size;
|
||||||
|
STD_TORCH_CHECK(scale.size(-1) == num_groups &&
|
||||||
|
scale.numel() == num_tokens * (int64_t)num_groups,
|
||||||
|
"situ_and_mul_quant: scale shape must be "
|
||||||
|
"[num_tokens, d/group_size]");
|
||||||
|
dim3 grid(GRID_DIM);
|
||||||
|
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||||
|
input.scalar_type(), "situ_and_mul_quant_group_kernel", [&] {
|
||||||
|
VLLM_STABLE_DISPATCH_FP8_TYPES(
|
||||||
|
out.scalar_type(), "situ_and_mul_quant_group_kernel_fp8", [&] {
|
||||||
|
#ifndef USE_ROCM
|
||||||
|
// The pipelined kernel's float2-per-lane geometry assumes a
|
||||||
|
// 32-lane warp (GROUP_SIZE == 4 * WARP_SIZE); on HIP (64-lane)
|
||||||
|
// fall back to the WARP_SIZE-generic scalar kernel.
|
||||||
|
constexpr int THREADS = 256;
|
||||||
|
constexpr int SITU_D = 3072; // Kimi-K3 fused w2 input dim
|
||||||
|
if constexpr (sizeof(scalar_t) == 2) {
|
||||||
|
if (d == SITU_D && (float)beta == vllm::SITU_BETA &&
|
||||||
|
(float)linear_beta == vllm::SITU_LINEAR_BETA) {
|
||||||
|
constexpr int D = SITU_D;
|
||||||
|
constexpr int GROUP_STAGES = 4;
|
||||||
|
constexpr int NUM_WARPS = THREADS / 32;
|
||||||
|
constexpr int STAGE_ELTS = 2 * 128;
|
||||||
|
dim3 block(THREADS);
|
||||||
|
size_t smem_bytes = (size_t)NUM_WARPS * GROUP_STAGES *
|
||||||
|
STAGE_ELTS * sizeof(scalar_t);
|
||||||
|
vllm::situ_and_mul_quant_group_pipelined_kernel<
|
||||||
|
scalar_t, fp8_t, THREADS, GROUP_STAGES, 128, GRID_DIM,
|
||||||
|
D><<<grid, block, smem_bytes, stream>>>(
|
||||||
|
out.mutable_data_ptr<fp8_t>(),
|
||||||
|
scale.mutable_data_ptr<float>(),
|
||||||
|
input.const_data_ptr<scalar_t>(), num_tokens,
|
||||||
|
valid_rows_ptr, topk);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
const int num_warps = std::min(num_groups, 1024 / WARP_SIZE);
|
||||||
|
dim3 block(num_warps * WARP_SIZE);
|
||||||
|
vllm::situ_and_mul_quant_group_scalar_kernel<scalar_t, fp8_t,
|
||||||
|
128>
|
||||||
|
<<<grid, block, 0, stream>>>(
|
||||||
|
out.mutable_data_ptr<fp8_t>(),
|
||||||
|
scale.mutable_data_ptr<float>(),
|
||||||
|
input.const_data_ptr<scalar_t>(), d, num_groups,
|
||||||
|
(float)beta, (float)linear_beta, num_tokens,
|
||||||
|
valid_rows_ptr, topk);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
namespace vllm {
|
namespace vllm {
|
||||||
|
|
||||||
// Element-wise activation kernel template.
|
// Element-wise activation kernel template.
|
||||||
|
|||||||
@@ -526,8 +526,16 @@ void fatrelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
|||||||
double threshold);
|
double threshold);
|
||||||
void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||||
double alpha = 1.702, double limit = 7.0);
|
double alpha = 1.702, double limit = 7.0);
|
||||||
void situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
void situ_and_mul(
|
||||||
double beta = 1.0, double linear_beta = -1.0);
|
torch::stable::Tensor& out, torch::stable::Tensor& input, double beta = 1.0,
|
||||||
|
double linear_beta = -1.0,
|
||||||
|
std::optional<torch::stable::Tensor> valid_rows = std::nullopt);
|
||||||
|
void situ_and_mul_quant(
|
||||||
|
torch::stable::Tensor& out, torch::stable::Tensor& scale,
|
||||||
|
torch::stable::Tensor& input, double beta = 1.0, double linear_beta = -1.0,
|
||||||
|
int64_t group_size = 0,
|
||||||
|
std::optional<torch::stable::Tensor> valid_rows = std::nullopt,
|
||||||
|
int64_t topk = 1);
|
||||||
void masked_situ_and_mul(torch::stable::Tensor& out,
|
void masked_situ_and_mul(torch::stable::Tensor& out,
|
||||||
torch::stable::Tensor& input,
|
torch::stable::Tensor& input,
|
||||||
const torch::stable::Tensor& expert_num_tokens,
|
const torch::stable::Tensor& expert_num_tokens,
|
||||||
|
|||||||
@@ -611,7 +611,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
|||||||
// SituGLU implementation used in Kimi models.
|
// SituGLU implementation used in Kimi models.
|
||||||
ops.def(
|
ops.def(
|
||||||
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
|
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
|
||||||
"linear_beta=-1.0) -> ()");
|
"linear_beta=-1.0, Tensor? valid_rows=None) -> ()");
|
||||||
|
// Fused SituGLU activation + dynamic FP8 quantization for the Humming w2 path
|
||||||
|
// (writes the fp8 down input and its float32 scale). group_size=0 ->
|
||||||
|
// per-token scale [.., 1]; group_size=128 -> k-major block-FP8 scale [..,
|
||||||
|
// d/128].
|
||||||
|
ops.def(
|
||||||
|
"situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, "
|
||||||
|
"float beta=1.0, float linear_beta=-1.0, int group_size=0, "
|
||||||
|
"Tensor? valid_rows=None, int topk=1) -> ()");
|
||||||
ops.def(
|
ops.def(
|
||||||
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
|
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
|
||||||
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
|
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
|
||||||
@@ -830,6 +838,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
|||||||
ops.impl("fatrelu_and_mul", TORCH_BOX(&fatrelu_and_mul));
|
ops.impl("fatrelu_and_mul", TORCH_BOX(&fatrelu_and_mul));
|
||||||
ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul));
|
ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul));
|
||||||
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
|
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
|
||||||
|
ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant));
|
||||||
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
|
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
|
||||||
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
|
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
|
||||||
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
|
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
model_name: "nm-testing/Qwen3-30B-A3B-MXFP4A16"
|
||||||
|
accuracy_threshold: 0.86
|
||||||
|
num_questions: 1319
|
||||||
|
num_fewshot: 5
|
||||||
|
server_args: >-
|
||||||
|
--enforce-eager
|
||||||
|
--max-model-len 8192
|
||||||
|
--quantization humming
|
||||||
|
--linear-backend humming
|
||||||
|
env:
|
||||||
|
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3","group_size":128}'
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
gpt-oss-20b-humming-act-fp8.yaml
|
gpt-oss-20b-humming-act-fp8.yaml
|
||||||
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
|
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
|
||||||
|
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8-block.yaml
|
||||||
Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
|
Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
|
||||||
Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
|
Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
|
||||||
Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
|
Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
|
||||||
|
|||||||
@@ -1408,6 +1408,7 @@ def test_humming_delegates_to_instance_activation():
|
|||||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||||
input=input,
|
input=input,
|
||||||
output=output,
|
output=output,
|
||||||
|
valid_rows=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,406 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Tests for the fused MoE topk weight-and-reduce kernel.
|
||||||
|
|
||||||
|
Run `pytest tests/kernels/moe/test_moe_fused_mul_sum.py`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.model_executor.layers.fused_moe.moe_fused_mul_sum import (
|
||||||
|
moe_fused_mul_sum,
|
||||||
|
)
|
||||||
|
from vllm.utils.torch_utils import set_random_seed
|
||||||
|
|
||||||
|
NUM_TOKENS = [1, 17, 256]
|
||||||
|
TOP_KS = [2, 8]
|
||||||
|
HIDDEN_SIZE = 512
|
||||||
|
# One BLOCK_K tile, a partial trailing tile, and Kimi-K3's real dims
|
||||||
|
# (moe_intermediate_size 3072, hidden_size 7168) -- exercises the per-tile loop
|
||||||
|
# and its tail mask (BLOCK_K caps at 512 for <=2-byte dtypes).
|
||||||
|
HIDDEN_SIZES = [512, 1000, 3072, 7168]
|
||||||
|
DTYPES = [torch.bfloat16, torch.float32]
|
||||||
|
NUM_EXPERTS = 16
|
||||||
|
|
||||||
|
|
||||||
|
def _reference(
|
||||||
|
inputs: torch.Tensor,
|
||||||
|
topk_weights: torch.Tensor,
|
||||||
|
topk_ids: torch.Tensor,
|
||||||
|
expert_map: torch.Tensor | None,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
valid = topk_ids >= 0
|
||||||
|
if expert_map is not None:
|
||||||
|
valid &= expert_map[topk_ids.clamp(min=0)] >= 0
|
||||||
|
weights = torch.where(valid, topk_weights, torch.zeros_like(topk_weights))
|
||||||
|
return (inputs.float() * weights.float().unsqueeze(-1)).sum(dim=1)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||||
|
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||||
|
@pytest.mark.parametrize("use_expert_map", [True, False])
|
||||||
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||||
|
def test_invalid_slots_excluded(num_tokens: int, top_k: int, use_expert_map: bool):
|
||||||
|
"""Slots marked -1 must not contribute, and must not be dereferenced.
|
||||||
|
|
||||||
|
The expert GEMM never writes rows for topk slots whose expert id is -1
|
||||||
|
(a non-local expert under EP, or an all2all padding row), so those rows
|
||||||
|
hold stale workspace data. Filling them with NaN here means any failure to
|
||||||
|
mask them shows up as NaN in the output rather than a small numeric drift.
|
||||||
|
Indexing `expert_map` with -1 is also an out-of-bounds read.
|
||||||
|
"""
|
||||||
|
set_random_seed(0)
|
||||||
|
device = "cuda"
|
||||||
|
|
||||||
|
inputs = torch.randn(
|
||||||
|
num_tokens, top_k, HIDDEN_SIZE, dtype=torch.bfloat16, device=device
|
||||||
|
)
|
||||||
|
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
|
||||||
|
topk_ids = torch.randint(
|
||||||
|
0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
|
||||||
|
# Mark roughly half the slots invalid and poison the rows behind them.
|
||||||
|
invalid = torch.rand(num_tokens, top_k, device=device) < 0.5
|
||||||
|
topk_ids[invalid] = -1
|
||||||
|
inputs[invalid] = float("nan")
|
||||||
|
|
||||||
|
expert_map = None
|
||||||
|
if use_expert_map:
|
||||||
|
# Second half of the experts is non-local, so valid ids can still map
|
||||||
|
# to -1 -- exercising the second masking condition.
|
||||||
|
expert_map = torch.full((NUM_EXPERTS,), -1, dtype=torch.int32, device=device)
|
||||||
|
local = NUM_EXPERTS // 2
|
||||||
|
expert_map[:local] = torch.arange(local, dtype=torch.int32, device=device)
|
||||||
|
inputs[(topk_ids >= 0) & (expert_map[topk_ids.clamp(min=0)] < 0)] = float("nan")
|
||||||
|
|
||||||
|
# The _reference implementation zeros out outputs. We need to match that here,
|
||||||
|
# otherwise the test fails.
|
||||||
|
outputs = torch.zeros(num_tokens, HIDDEN_SIZE, dtype=torch.bfloat16, device=device)
|
||||||
|
out = moe_fused_mul_sum(
|
||||||
|
inputs=inputs,
|
||||||
|
topk_weights=topk_weights,
|
||||||
|
outputs=outputs,
|
||||||
|
topk_ids=topk_ids,
|
||||||
|
expert_map=expert_map,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not out.isnan().any(), "invalid slots leaked into the reduction"
|
||||||
|
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map)
|
||||||
|
torch.testing.assert_close(out.float(), ref, atol=1e-2, rtol=1e-2)
|
||||||
|
|
||||||
|
|
||||||
|
def _local_expert_map(device: str) -> torch.Tensor:
|
||||||
|
"""First half of the experts is local, second half maps to -1 (non-local)."""
|
||||||
|
expert_map = torch.full((NUM_EXPERTS,), -1, dtype=torch.int32, device=device)
|
||||||
|
local = NUM_EXPERTS // 2
|
||||||
|
expert_map[:local] = torch.arange(local, dtype=torch.int32, device=device)
|
||||||
|
return expert_map
|
||||||
|
|
||||||
|
|
||||||
|
SENTINEL = 42.0 # exact in bf16; marks output the kernel must leave untouched
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||||
|
def test_nonlocal_row_is_zeroed():
|
||||||
|
"""A real row whose positive ids all map to non-local experts is written 0.
|
||||||
|
|
||||||
|
Such a row is not padding (its ids are >= 0, so row_valid > 0 and the block
|
||||||
|
is not early-returned). Every slot is then masked out by expert_map, so the
|
||||||
|
accumulator stays 0 and 0 is stored -- the row must not be left holding stale
|
||||||
|
buffer data, because downstream reduction treats it as a real token.
|
||||||
|
"""
|
||||||
|
set_random_seed(0)
|
||||||
|
device = "cuda"
|
||||||
|
num_tokens, top_k = 17, 8
|
||||||
|
local = NUM_EXPERTS // 2
|
||||||
|
expert_map = _local_expert_map(device)
|
||||||
|
|
||||||
|
# Poison inputs so any failure to mask the non-local slots surfaces as NaN.
|
||||||
|
inputs = torch.full(
|
||||||
|
(num_tokens, top_k, HIDDEN_SIZE),
|
||||||
|
float("nan"),
|
||||||
|
dtype=torch.bfloat16,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
|
||||||
|
# Every id lands in the non-local half [local, NUM_EXPERTS).
|
||||||
|
topk_ids = torch.randint(
|
||||||
|
local, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
|
||||||
|
outputs = torch.full(
|
||||||
|
(num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device
|
||||||
|
)
|
||||||
|
out = moe_fused_mul_sum(
|
||||||
|
inputs=inputs,
|
||||||
|
topk_weights=topk_weights,
|
||||||
|
outputs=outputs,
|
||||||
|
topk_ids=topk_ids,
|
||||||
|
expert_map=expert_map,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not out.isnan().any(), "non-local row leaked NaN into the output"
|
||||||
|
torch.testing.assert_close(out.float(), torch.zeros_like(out.float()))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("use_expert_map", [True, False])
|
||||||
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||||
|
def test_padding_row_left_untouched(use_expert_map: bool):
|
||||||
|
"""An all -1 padding row is skipped: its output row is left untouched.
|
||||||
|
|
||||||
|
This is the CUDA-graph decode contract -- padding rows past num_recv are
|
||||||
|
never read downstream, so the kernel elides them (whole-padding blocks
|
||||||
|
early-return) rather than zeroing. The skip is gated on topk_ids alone, so
|
||||||
|
it must hold whether or not expert_map is passed (the humming path passes
|
||||||
|
None); we pin both.
|
||||||
|
"""
|
||||||
|
set_random_seed(0)
|
||||||
|
device = "cuda"
|
||||||
|
num_tokens, top_k = 17, 8
|
||||||
|
expert_map = _local_expert_map(device) if use_expert_map else None
|
||||||
|
|
||||||
|
inputs = torch.full(
|
||||||
|
(num_tokens, top_k, HIDDEN_SIZE),
|
||||||
|
float("nan"),
|
||||||
|
dtype=torch.bfloat16,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
|
||||||
|
topk_ids = torch.full((num_tokens, top_k), -1, dtype=torch.int32, device=device)
|
||||||
|
|
||||||
|
outputs = torch.full(
|
||||||
|
(num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device
|
||||||
|
)
|
||||||
|
out = moe_fused_mul_sum(
|
||||||
|
inputs=inputs,
|
||||||
|
topk_weights=topk_weights,
|
||||||
|
outputs=outputs,
|
||||||
|
topk_ids=topk_ids,
|
||||||
|
expert_map=expert_map,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert torch.all(out == SENTINEL), "padding row output was modified"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||||
|
def test_nonlocal_and_padding_adjacent():
|
||||||
|
"""A non-local real row next to an all -1 padding row.
|
||||||
|
|
||||||
|
Each CTA owns one token row: the non-local row must be zeroed (its slots are
|
||||||
|
all masked by expert_map) while the padding row beside it is left untouched.
|
||||||
|
Guards against a change that lets one row's fate leak into its neighbour.
|
||||||
|
"""
|
||||||
|
set_random_seed(0)
|
||||||
|
device = "cuda"
|
||||||
|
num_tokens, top_k = 17, 8
|
||||||
|
local = NUM_EXPERTS // 2
|
||||||
|
expert_map = _local_expert_map(device)
|
||||||
|
|
||||||
|
inputs = torch.full(
|
||||||
|
(num_tokens, top_k, HIDDEN_SIZE),
|
||||||
|
float("nan"),
|
||||||
|
dtype=torch.bfloat16,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
|
||||||
|
# All rows are non-local real rows except row 1, an all -1 padding row.
|
||||||
|
topk_ids = torch.randint(
|
||||||
|
local, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
topk_ids[1] = -1
|
||||||
|
|
||||||
|
outputs = torch.full(
|
||||||
|
(num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device
|
||||||
|
)
|
||||||
|
out = moe_fused_mul_sum(
|
||||||
|
inputs=inputs,
|
||||||
|
topk_weights=topk_weights,
|
||||||
|
outputs=outputs,
|
||||||
|
topk_ids=topk_ids,
|
||||||
|
expert_map=expert_map,
|
||||||
|
)
|
||||||
|
|
||||||
|
padding = torch.zeros(num_tokens, dtype=torch.bool, device=device)
|
||||||
|
padding[1] = True
|
||||||
|
assert torch.all(out[1] == SENTINEL), "padding row beside a real row was modified"
|
||||||
|
written = out[~padding].float()
|
||||||
|
assert not written.isnan().any(), "non-local row leaked NaN into the output"
|
||||||
|
torch.testing.assert_close(written, torch.zeros_like(written))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||||
|
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||||
|
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||||
|
@pytest.mark.parametrize("dtype", DTYPES)
|
||||||
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||||
|
def test_weighted_sum_matches_reference(
|
||||||
|
num_tokens: int, top_k: int, hidden_size: int, dtype: torch.dtype
|
||||||
|
):
|
||||||
|
"""Weighted sum over live slots matches the reference across hidden tiles.
|
||||||
|
|
||||||
|
Covers the deployed humming path (globalized ids, expert_map=None): a random
|
||||||
|
-1 mix with NaN-poisoned dead slots, over hidden sizes that span a single
|
||||||
|
tile, a partial trailing tile, and many tiles, in both bf16 and fp32.
|
||||||
|
"""
|
||||||
|
set_random_seed(0)
|
||||||
|
device = "cuda"
|
||||||
|
|
||||||
|
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||||
|
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||||
|
topk_ids = torch.randint(
|
||||||
|
0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
invalid = torch.rand(num_tokens, top_k, device=device) < 0.5
|
||||||
|
topk_ids[invalid] = -1
|
||||||
|
inputs[invalid] = float("nan")
|
||||||
|
|
||||||
|
# All-padding rows early-return untouched, so pre-zero to match the reference.
|
||||||
|
outputs = torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||||
|
out = moe_fused_mul_sum(
|
||||||
|
inputs=inputs,
|
||||||
|
topk_weights=topk_weights,
|
||||||
|
outputs=outputs,
|
||||||
|
topk_ids=topk_ids,
|
||||||
|
expert_map=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not out.isnan().any(), "invalid slots leaked into the reduction"
|
||||||
|
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map=None)
|
||||||
|
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||||
|
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("dtype", DTYPES)
|
||||||
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||||
|
def test_no_topk_ids_sums_all_slots(dtype: torch.dtype):
|
||||||
|
"""With topk_ids=None every slot is live: plain weighted sum over top_k."""
|
||||||
|
set_random_seed(0)
|
||||||
|
device = "cuda"
|
||||||
|
num_tokens, top_k, hidden_size = 17, 8, 1000
|
||||||
|
|
||||||
|
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||||
|
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||||
|
out = moe_fused_mul_sum(inputs=inputs, topk_weights=topk_weights)
|
||||||
|
|
||||||
|
ref = (inputs.float() * topk_weights.float().unsqueeze(-1)).sum(dim=1)
|
||||||
|
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||||
|
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("hidden_size", [512, 7168])
|
||||||
|
@pytest.mark.parametrize("dtype", DTYPES)
|
||||||
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||||
|
def test_expert_map_is_required_for_nonlocal_ids(hidden_size: int, dtype: torch.dtype):
|
||||||
|
"""expert_map is load-bearing, not redundant: dropping it leaks non-local ids.
|
||||||
|
|
||||||
|
A positive id whose expert is non-local marks a slot the GEMM never wrote, so
|
||||||
|
its `inputs` row is stale. `topk_ids >= 0` alone cannot tell it apart from a
|
||||||
|
real slot -- only expert_map can. This pins the contract the humming path
|
||||||
|
relies on (globalized ids are still routed through expert_map as a safety
|
||||||
|
net): with the map the stale slot is masked; without it, it corrupts the sum.
|
||||||
|
Regression guard against re-dropping expert_map as an optimization.
|
||||||
|
"""
|
||||||
|
set_random_seed(0)
|
||||||
|
device = "cuda"
|
||||||
|
num_tokens, top_k = 8, 8
|
||||||
|
local = NUM_EXPERTS // 2
|
||||||
|
expert_map = _local_expert_map(device)
|
||||||
|
|
||||||
|
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||||
|
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||||
|
# Each row keeps at least one local slot (so the mapped output is finite) and
|
||||||
|
# at least one non-local slot whose stale row is poisoned with NaN.
|
||||||
|
half = top_k // 2
|
||||||
|
topk_ids = torch.empty(num_tokens, top_k, dtype=torch.int32, device=device)
|
||||||
|
topk_ids[:, :half] = torch.randint(
|
||||||
|
0, local, (num_tokens, half), dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
topk_ids[:, half:] = torch.randint(
|
||||||
|
local, NUM_EXPERTS, (num_tokens, top_k - half), dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
inputs[:, half:] = float("nan")
|
||||||
|
|
||||||
|
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||||
|
|
||||||
|
mapped = moe_fused_mul_sum(
|
||||||
|
inputs=inputs,
|
||||||
|
topk_weights=topk_weights,
|
||||||
|
outputs=torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device),
|
||||||
|
topk_ids=topk_ids,
|
||||||
|
expert_map=expert_map,
|
||||||
|
)
|
||||||
|
assert not mapped.isnan().any(), "expert_map failed to mask stale non-local slot"
|
||||||
|
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map)
|
||||||
|
torch.testing.assert_close(mapped.float(), ref, atol=atol, rtol=rtol)
|
||||||
|
|
||||||
|
# Same call without the map: the non-local slot is summed and the NaN leaks.
|
||||||
|
unmapped = moe_fused_mul_sum(
|
||||||
|
inputs=inputs,
|
||||||
|
topk_weights=topk_weights,
|
||||||
|
outputs=torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device),
|
||||||
|
topk_ids=topk_ids,
|
||||||
|
expert_map=None,
|
||||||
|
)
|
||||||
|
assert unmapped.isnan().any(), "expert_map is not redundant: dropping it must leak"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("dtype", DTYPES)
|
||||||
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||||
|
def test_num_valid_tokens_bounds_stale_padding(dtype: torch.dtype):
|
||||||
|
"""Rows past num_valid_tokens are left untouched, even with stale ids/NaN.
|
||||||
|
|
||||||
|
Under cudagraph decode the grid is the static padded token count and the
|
||||||
|
tail rows past num_recv hold stale (non -1) ids and stale NaN in `inputs`
|
||||||
|
from a prior replay -- the -1 mask alone cannot spot them. Bounding on
|
||||||
|
num_valid_tokens must skip the tail before any load so its NaN never reaches
|
||||||
|
the output; without the bound it is summed and a masked finalize can then
|
||||||
|
propagate the NaN. Regression guard for the decode accuracy fix.
|
||||||
|
"""
|
||||||
|
set_random_seed(0)
|
||||||
|
device = "cuda"
|
||||||
|
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 512
|
||||||
|
|
||||||
|
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||||
|
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||||
|
# All ids valid (>= 0): the stale tail is indistinguishable by the -1 mask.
|
||||||
|
topk_ids = torch.randint(
|
||||||
|
0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
inputs[num_recv:] = float("nan") # stale down_output behind the padding tail
|
||||||
|
num_valid = torch.tensor([num_recv], dtype=torch.int32, device=device)
|
||||||
|
|
||||||
|
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||||
|
|
||||||
|
bounded = moe_fused_mul_sum(
|
||||||
|
inputs=inputs,
|
||||||
|
topk_weights=topk_weights,
|
||||||
|
outputs=torch.full(
|
||||||
|
(num_tokens, hidden_size), SENTINEL, dtype=dtype, device=device
|
||||||
|
),
|
||||||
|
topk_ids=topk_ids,
|
||||||
|
num_valid_tokens=num_valid,
|
||||||
|
)
|
||||||
|
assert torch.all(bounded[num_recv:] == SENTINEL), (
|
||||||
|
"padding tail past num_recv written"
|
||||||
|
)
|
||||||
|
assert not bounded[:num_recv].isnan().any(), "real rows leaked NaN"
|
||||||
|
ref = _reference(
|
||||||
|
inputs[:num_recv].nan_to_num(),
|
||||||
|
topk_weights[:num_recv],
|
||||||
|
topk_ids[:num_recv],
|
||||||
|
expert_map=None,
|
||||||
|
)
|
||||||
|
torch.testing.assert_close(bounded[:num_recv].float(), ref, atol=atol, rtol=rtol)
|
||||||
|
|
||||||
|
# Without the bound the stale tail is summed and leaks NaN into its rows.
|
||||||
|
unbounded = moe_fused_mul_sum(
|
||||||
|
inputs=inputs,
|
||||||
|
topk_weights=topk_weights,
|
||||||
|
outputs=torch.full(
|
||||||
|
(num_tokens, hidden_size), SENTINEL, dtype=dtype, device=device
|
||||||
|
),
|
||||||
|
topk_ids=topk_ids,
|
||||||
|
)
|
||||||
|
assert unbounded[num_recv:].isnan().any(), "num_valid_tokens bound is load-bearing"
|
||||||
@@ -0,0 +1,224 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Correctness tests for the persistent cp.async-pipelined SITU + block-FP8
|
||||||
|
quant kernel (``situ_and_mul_quant``).
|
||||||
|
|
||||||
|
Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference
|
||||||
|
activation + block-FP8 quant, drive random per-row scale magnitudes, exercise
|
||||||
|
the masked (``valid_rows``) contiguous layout, and compare per valid row.
|
||||||
|
|
||||||
|
The kernel computes SITU in fp32 and converts to fp8 with hardware rounding,
|
||||||
|
which torch cannot reproduce bit-exactly. It is therefore validated against the
|
||||||
|
*ideal* FP8 quantizer of the exact fp64 activation: the difference must stay
|
||||||
|
within the unavoidable FP8 block-quant error (within 1 fp8 code for ~all
|
||||||
|
elements, never off by more than 2).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.model_executor.layers.fused_moe.activation import situ_and_mul_quant
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
from vllm.utils.torch_utils import set_random_seed
|
||||||
|
|
||||||
|
fp8_dtype = current_platform.fp8_dtype()
|
||||||
|
DEVICE = current_platform.device_type
|
||||||
|
FP8_MAX = torch.finfo(fp8_dtype).max
|
||||||
|
|
||||||
|
# The kernel uses fp8 e4m3 conversion, so it is CUDA-only and needs SM89+
|
||||||
|
# (Ada/Hopper). There is no Triton fallback.
|
||||||
|
pytestmark = pytest.mark.skipif(
|
||||||
|
not current_platform.is_cuda() or not current_platform.has_device_capability(89),
|
||||||
|
reason="situ_and_mul_quant persistent kernel requires CUDA SM89+ (fp8 e4m3).",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Kimi-K3 baked SITU params. Only (d=3072, beta=4.0, linear_beta=25.0,
|
||||||
|
# group_size=128, fp16/bf16) routes to the persistent cp.async-pipelined kernel;
|
||||||
|
# anything else falls back to the scalar group kernel.
|
||||||
|
SITU_BETA = 4.0
|
||||||
|
SITU_LINEAR_BETA = 25.0
|
||||||
|
SITU_D = 3072
|
||||||
|
GROUP_SIZE = 128
|
||||||
|
|
||||||
|
# (num_tokens, valid_rows). valid_rows=None => all rows valid. 2048/4096 rows
|
||||||
|
# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8).
|
||||||
|
CASES = [
|
||||||
|
(1, None),
|
||||||
|
(17, None),
|
||||||
|
(128, None),
|
||||||
|
(2048, None),
|
||||||
|
(4096, None),
|
||||||
|
(2048, 1500), # DeepEP v2 contiguous padding
|
||||||
|
(2048, 1),
|
||||||
|
(4096, 4095),
|
||||||
|
(512, 0), # whole tensor is padding
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def token_random(num_tokens: int, twod: int, dtype: torch.dtype) -> torch.Tensor:
|
||||||
|
"""Per-row random exponent so groups span many scale magnitudes."""
|
||||||
|
base = torch.randn(num_tokens, twod, dtype=torch.float32, device=DEVICE)
|
||||||
|
exps = torch.randint(1, 13, (num_tokens, 1), device=DEVICE).float()
|
||||||
|
return (base * (2.0**exps)).to(dtype)
|
||||||
|
|
||||||
|
|
||||||
|
def ref_situ_block_fp8_quant(
|
||||||
|
inp: torch.Tensor, beta: float, linear_beta: float, group_size: int
|
||||||
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
"""Ideal FP8 block-quant of the exact fp64 SITU activation.
|
||||||
|
|
||||||
|
The kernel rounds the activation to ``scalar_t`` before quantizing (it stores
|
||||||
|
``(float)(scalar_t)situ(...)``), so the reference does the same before taking
|
||||||
|
the per-group absmax.
|
||||||
|
"""
|
||||||
|
d = inp.shape[-1] // 2
|
||||||
|
gate, up = inp.double().chunk(2, dim=-1)
|
||||||
|
gate_out = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
|
||||||
|
up_out = linear_beta * torch.tanh(up / linear_beta) if linear_beta > 0 else up
|
||||||
|
act = (gate_out * up_out).to(inp.dtype).float()
|
||||||
|
|
||||||
|
m = act.shape[0]
|
||||||
|
ng = d // group_size
|
||||||
|
a = act.view(m, ng, group_size)
|
||||||
|
scale = a.abs().amax(dim=2).clamp_min(1e-30) / FP8_MAX
|
||||||
|
q = (a / scale.unsqueeze(2)).clamp(-FP8_MAX, FP8_MAX).to(fp8_dtype)
|
||||||
|
return q.view(m, d), scale
|
||||||
|
|
||||||
|
|
||||||
|
def fp8_code_dist(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
||||||
|
"""Signed-magnitude fp8 code distance (valid for e4m3fn and e4m3fnuz)."""
|
||||||
|
|
||||||
|
def ordered(x: torch.Tensor) -> torch.Tensor:
|
||||||
|
x = x.view(torch.uint8).to(torch.int32)
|
||||||
|
return torch.where((x >> 7) == 1, -(x & 0x7F), x & 0x7F)
|
||||||
|
|
||||||
|
return (ordered(a) - ordered(b)).abs()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("num_tokens,valid", CASES)
|
||||||
|
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
|
||||||
|
@torch.inference_mode()
|
||||||
|
def test_situ_and_mul_quant_pipelined(
|
||||||
|
num_tokens: int, valid: int | None, dtype: torch.dtype
|
||||||
|
) -> None:
|
||||||
|
set_random_seed(42)
|
||||||
|
d = SITU_D
|
||||||
|
inp = token_random(num_tokens, 2 * d, dtype)
|
||||||
|
|
||||||
|
# Sentinels: padding rows must keep their bytes (out) and get scale 1.0.
|
||||||
|
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
|
||||||
|
scale = torch.full(
|
||||||
|
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||||
|
)
|
||||||
|
out_sentinel = out.clone()
|
||||||
|
valid_rows = (
|
||||||
|
None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE)
|
||||||
|
)
|
||||||
|
|
||||||
|
situ_and_mul_quant(
|
||||||
|
out,
|
||||||
|
scale,
|
||||||
|
inp,
|
||||||
|
beta=SITU_BETA,
|
||||||
|
linear_beta=SITU_LINEAR_BETA,
|
||||||
|
group_size=GROUP_SIZE,
|
||||||
|
valid_rows=valid_rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Padding contract: skipped rows keep their bytes; their scale becomes 1.0.
|
||||||
|
if valid is not None and valid < num_tokens:
|
||||||
|
assert torch.equal(scale[valid:], torch.ones_like(scale[valid:]))
|
||||||
|
assert torch.equal(
|
||||||
|
out[valid:].view(torch.uint8), out_sentinel[valid:].view(torch.uint8)
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = num_tokens if valid is None else valid
|
||||||
|
if rows == 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
ref_q, ref_s = ref_situ_block_fp8_quant(
|
||||||
|
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
|
||||||
|
)
|
||||||
|
|
||||||
|
assert torch.isfinite(scale[:rows]).all()
|
||||||
|
assert not torch.isnan(out[:rows].float()).any()
|
||||||
|
|
||||||
|
# Scales: the kernel's fp32 SITU shifts the per-group absmax only
|
||||||
|
# marginally vs the fp64 reference.
|
||||||
|
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
|
||||||
|
|
||||||
|
# FP8 codes: must land within 1 code of the ideal quantizer for ~all
|
||||||
|
# elements, never off by more than 2.
|
||||||
|
dist = fp8_code_dist(out[:rows], ref_q)
|
||||||
|
assert dist.max().item() <= 2
|
||||||
|
assert (dist <= 1).float().mean().item() >= 0.99
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)])
|
||||||
|
@torch.inference_mode()
|
||||||
|
def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
|
||||||
|
"""valid_rows is a token count; the kernel bounds rows at tokens*topk."""
|
||||||
|
set_random_seed(7)
|
||||||
|
d = SITU_D
|
||||||
|
num_tokens = tokens * topk + 37 # padding tail past the valid rows
|
||||||
|
inp = token_random(num_tokens, 2 * d, torch.bfloat16)
|
||||||
|
|
||||||
|
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
|
||||||
|
scale = torch.full(
|
||||||
|
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||||
|
)
|
||||||
|
out_sentinel = out.clone()
|
||||||
|
valid_rows = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
|
||||||
|
|
||||||
|
situ_and_mul_quant(
|
||||||
|
out,
|
||||||
|
scale,
|
||||||
|
inp,
|
||||||
|
beta=SITU_BETA,
|
||||||
|
linear_beta=SITU_LINEAR_BETA,
|
||||||
|
group_size=GROUP_SIZE,
|
||||||
|
valid_rows=valid_rows,
|
||||||
|
topk=topk,
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = tokens * topk
|
||||||
|
assert torch.equal(scale[rows:], torch.ones_like(scale[rows:]))
|
||||||
|
assert torch.equal(
|
||||||
|
out[rows:].view(torch.uint8), out_sentinel[rows:].view(torch.uint8)
|
||||||
|
)
|
||||||
|
|
||||||
|
ref_q, ref_s = ref_situ_block_fp8_quant(
|
||||||
|
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
|
||||||
|
)
|
||||||
|
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
|
||||||
|
assert fp8_code_dist(out[:rows], ref_q).max().item() <= 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
|
||||||
|
@torch.inference_mode()
|
||||||
|
def test_situ_and_mul_quant_pipelined_deterministic(dtype: torch.dtype) -> None:
|
||||||
|
"""Two runs on the same input must be byte-identical."""
|
||||||
|
set_random_seed(0)
|
||||||
|
d = SITU_D
|
||||||
|
num_tokens = 2048
|
||||||
|
inp = token_random(num_tokens, 2 * d, dtype)
|
||||||
|
|
||||||
|
def run() -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
out = torch.empty(num_tokens, d, dtype=fp8_dtype, device=DEVICE)
|
||||||
|
scale = torch.empty(
|
||||||
|
num_tokens, d // GROUP_SIZE, dtype=torch.float32, device=DEVICE
|
||||||
|
)
|
||||||
|
situ_and_mul_quant(
|
||||||
|
out,
|
||||||
|
scale,
|
||||||
|
inp,
|
||||||
|
beta=SITU_BETA,
|
||||||
|
linear_beta=SITU_LINEAR_BETA,
|
||||||
|
group_size=GROUP_SIZE,
|
||||||
|
)
|
||||||
|
return out, scale
|
||||||
|
|
||||||
|
out1, s1 = run()
|
||||||
|
out2, s2 = run()
|
||||||
|
assert torch.equal(out1.view(torch.uint8), out2.view(torch.uint8))
|
||||||
|
assert torch.equal(s1, s2)
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Tests for Humming MXFP4-weight GEMM paired with a block-FP8 activation.
|
||||||
|
|
||||||
|
The Humming kernel already implements MXFP4 weight (group-32 e8m0) with a
|
||||||
|
grouped FP8 activation via WGMMA software dequant -- see the grouped-fp8
|
||||||
|
`test_mxfp4.py` cases in the humming repo, which run on SM90 (H200). Enabling
|
||||||
|
it in vLLM comes down to two things, both exercised here:
|
||||||
|
|
||||||
|
1. A grouped FP8 activation must classify to a float32-scaled key
|
||||||
|
(kFp8Dynamic128Sym), not a uint8 microscale key. MX microscale
|
||||||
|
activations (group size 32) keep their e8m0 (uint8) scale.
|
||||||
|
2. (kMxfp4Static, kFp8Dynamic128Sym) must be an allowed MoE
|
||||||
|
(weight, activation) pair.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||||
|
HummingExpertsBase,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||||
|
kFp8Dynamic128Sym,
|
||||||
|
kFp8DynamicTokenSym,
|
||||||
|
kMxfp4Static,
|
||||||
|
kMxfp8Dynamic,
|
||||||
|
)
|
||||||
|
from vllm.utils.import_utils import has_humming
|
||||||
|
|
||||||
|
# make_humming_moe_quant_config stores per-sublayer humming LayerConfigs but the
|
||||||
|
# config-plumbing asserts below never read them, so a placeholder satisfies the
|
||||||
|
# required arg without pulling in the humming package.
|
||||||
|
_PLACEHOLDER_HUMMING_CONFIGS = {"w13": None, "w2": None}
|
||||||
|
|
||||||
|
|
||||||
|
def test_mxfp4_weight_with_block_fp8_activation_is_supported():
|
||||||
|
"""MXFP4 weight + block-FP8 (group-128) activation is an allowed pair."""
|
||||||
|
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8Dynamic128Sym)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mxfp4_weight_with_per_token_fp8_activation_still_supported():
|
||||||
|
"""The pre-existing per-token FP8 pairing must keep working."""
|
||||||
|
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8DynamicTokenSym)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"group_size,expected_key",
|
||||||
|
[
|
||||||
|
# per-token FP8 -> float32 per-token scale
|
||||||
|
(0, kFp8DynamicTokenSym),
|
||||||
|
# block FP8 -> float32 group-128 scale (the newly-fixed case)
|
||||||
|
(128, kFp8Dynamic128Sym),
|
||||||
|
# MXFP8 microscale -> e8m0 (uint8) group-32 scale (must be preserved)
|
||||||
|
(32, kMxfp8Dynamic),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_humming_fp8_input_schema_to_quant_key(group_size, expected_key):
|
||||||
|
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||||
|
_humming_input_schema_to_quant_key,
|
||||||
|
)
|
||||||
|
from vllm.utils.humming import HummingInputSchema
|
||||||
|
from vllm.utils.humming import dtypes as humming_dtypes
|
||||||
|
|
||||||
|
schema = HummingInputSchema(
|
||||||
|
a_dtype=humming_dtypes.float8e4m3,
|
||||||
|
input_scale_group_size=group_size,
|
||||||
|
)
|
||||||
|
assert _humming_input_schema_to_quant_key(schema) == expected_key
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
|
||||||
|
def test_humming_bf16_input_schema_is_unquantized():
|
||||||
|
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||||
|
_humming_input_schema_to_quant_key,
|
||||||
|
)
|
||||||
|
from vllm.utils.humming import HummingInputSchema
|
||||||
|
|
||||||
|
# No a_dtype -> unquantized (bf16/fp16) inputs -> None.
|
||||||
|
assert _humming_input_schema_to_quant_key(HummingInputSchema()) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_block_fp8_activation_quant_config_is_block_quantized():
|
||||||
|
"""A block-FP8 activation group shape yields a block-quantized MoE config
|
||||||
|
(FP8 dtype, [1, 128] block, not per-act-token). This is what makes the
|
||||||
|
DeepEP prepare/finalize step quantize activations to block FP8 *before* the
|
||||||
|
all-to-all dispatch instead of deferring to Humming -- and what makes
|
||||||
|
HummingExpertsBase.expects_unquantized_inputs return False."""
|
||||||
|
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||||
|
make_humming_moe_quant_config,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
|
qc = make_humming_moe_quant_config(
|
||||||
|
quant_dtype=current_platform.fp8_dtype(),
|
||||||
|
weight_dtype="float4e2m1",
|
||||||
|
activation_group_shape=GroupShape(row=1, col=128),
|
||||||
|
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
|
||||||
|
)
|
||||||
|
assert qc.is_block_quantized
|
||||||
|
assert qc.block_shape == [1, 128]
|
||||||
|
assert not qc.per_act_token_quant
|
||||||
|
assert qc.quant_dtype == current_platform.fp8_dtype()
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_activation_quant_config_defers_to_humming():
|
||||||
|
"""Without a block activation shape the config stays per-token (the deferred
|
||||||
|
path): Humming quantizes internally, preserving pre-existing behavior for
|
||||||
|
every non-block-FP8 scheme."""
|
||||||
|
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||||
|
make_humming_moe_quant_config,
|
||||||
|
)
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
|
qc = make_humming_moe_quant_config(
|
||||||
|
quant_dtype=current_platform.fp8_dtype(),
|
||||||
|
weight_dtype="float4e2m1",
|
||||||
|
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
|
||||||
|
)
|
||||||
|
assert not qc.is_block_quantized
|
||||||
|
assert qc.per_act_token_quant
|
||||||
@@ -201,6 +201,7 @@ def apply_moe_activation(
|
|||||||
activation_config: ApplyMoEActivationConfig | None = None,
|
activation_config: ApplyMoEActivationConfig | None = None,
|
||||||
topk_ids: torch.Tensor | None = None,
|
topk_ids: torch.Tensor | None = None,
|
||||||
expert_map: torch.Tensor | None = None,
|
expert_map: torch.Tensor | None = None,
|
||||||
|
valid_rows: torch.Tensor | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""Apply MoE activation function.
|
"""Apply MoE activation function.
|
||||||
|
|
||||||
@@ -256,6 +257,7 @@ def apply_moe_activation(
|
|||||||
-1.0
|
-1.0
|
||||||
if config.activation_situ_linear_beta is None
|
if config.activation_situ_linear_beta is None
|
||||||
else config.activation_situ_linear_beta,
|
else config.activation_situ_linear_beta,
|
||||||
|
valid_rows,
|
||||||
)
|
)
|
||||||
elif activation == MoEActivation.SWIGLUOAI:
|
elif activation == MoEActivation.SWIGLUOAI:
|
||||||
torch.ops._C.swigluoai_and_mul(output, input)
|
torch.ops._C.swigluoai_and_mul(output, input)
|
||||||
@@ -286,3 +288,45 @@ def apply_moe_activation(
|
|||||||
raise ValueError(f"Unsupported FusedMoe activation: {activation}")
|
raise ValueError(f"Unsupported FusedMoe activation: {activation}")
|
||||||
|
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
def situ_and_mul_quant(
|
||||||
|
output: torch.Tensor,
|
||||||
|
scale: torch.Tensor,
|
||||||
|
input: torch.Tensor,
|
||||||
|
*,
|
||||||
|
beta: float,
|
||||||
|
linear_beta: float | None,
|
||||||
|
group_size: int = 0,
|
||||||
|
valid_rows: torch.Tensor | None = None,
|
||||||
|
topk: int = 1,
|
||||||
|
) -> None:
|
||||||
|
"""Fused Kimi SITU activation + dynamic FP8 quantization.
|
||||||
|
|
||||||
|
Writes the quantized fp8 down-projection input into ``output`` [M, d] and its
|
||||||
|
float32 scale into ``scale`` (dequant = q * scale), fusing the SITU
|
||||||
|
activation with the FP8 quant that Humming's w2 GEMM would otherwise perform
|
||||||
|
in a separate pass -- this avoids materializing (and rounding through) the
|
||||||
|
bf16 activation_output buffer.
|
||||||
|
|
||||||
|
``group_size`` selects the quant granularity: ``0`` gives per-token scales
|
||||||
|
``scale`` [M, 1]; ``128`` gives k-major block-FP8 group scales ``scale``
|
||||||
|
[M, d // 128], matching humming ``quant_input(group_size=128, float32)``.
|
||||||
|
|
||||||
|
``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching
|
||||||
|
``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the
|
||||||
|
DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the
|
||||||
|
kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply
|
||||||
|
is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept
|
||||||
|
in this module so that every ``torch.ops._C.situ*`` call lives in one file.
|
||||||
|
"""
|
||||||
|
torch.ops._C.situ_and_mul_quant(
|
||||||
|
output,
|
||||||
|
scale,
|
||||||
|
input,
|
||||||
|
beta,
|
||||||
|
-1.0 if linear_beta is None else linear_beta,
|
||||||
|
group_size,
|
||||||
|
valid_rows,
|
||||||
|
topk,
|
||||||
|
)
|
||||||
|
|||||||
@@ -525,7 +525,9 @@ def deepgemm_moe_permute(
|
|||||||
expert_num_tokens = None
|
expert_num_tokens = None
|
||||||
if expert_tokens_meta is not None:
|
if expert_tokens_meta is not None:
|
||||||
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
||||||
else:
|
if expert_num_tokens is None:
|
||||||
|
# A cudagraph-decode carrier may hold only psum_recv_per_rank and leave
|
||||||
|
# expert_num_tokens None; count locally as in the meta-absent case.
|
||||||
expert_num_tokens = count_expert_num_tokens(
|
expert_num_tokens = count_expert_num_tokens(
|
||||||
topk_ids, local_num_experts, expert_map
|
topk_ids, local_num_experts, expert_map
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -668,6 +668,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
|
|||||||
assert hidden_states.dim() == 3
|
assert hidden_states.dim() == 3
|
||||||
assert expert_tokens_meta is not None
|
assert expert_tokens_meta is not None
|
||||||
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
||||||
|
assert expert_num_tokens is not None
|
||||||
|
|
||||||
num_local_experts = w1.size(0)
|
num_local_experts = w1.size(0)
|
||||||
assert num_local_experts == w1.size(0), f"{num_local_experts} == {w1.size(0)}"
|
assert num_local_experts == w1.size(0), f"{num_local_experts} == {w1.size(0)}"
|
||||||
|
|||||||
@@ -119,6 +119,37 @@ def get_humming_moe_gemm_type() -> str:
|
|||||||
return gemm_type
|
return gemm_type
|
||||||
|
|
||||||
|
|
||||||
|
def _fixup_moe_tuning_config(tuning_config: list, max_k_block: int = 128) -> None:
|
||||||
|
"""Fix up each MoE tile in place: cap the K-block and widen warp-N.
|
||||||
|
|
||||||
|
- block_shape[2] (K-block) > ``max_k_block``: the driver rejects the TMA
|
||||||
|
descriptor at launch (CUDA_ERROR_MISALIGNED_ADDRESS). Cap at 128, which
|
||||||
|
Humming already uses for larger M.
|
||||||
|
- warp_shape[1] (warp-N) < 32: block-FP8 (group-128) activations route the
|
||||||
|
w13 (gate/up) GEMM to a tuning table that pins warp-N to 16, under-filling
|
||||||
|
the Hopper WGMMA N dimension and corrupting the GEMM output (gsm8k
|
||||||
|
0.94 -> 0.89). Widen to 32 whenever block_n % 32 == 0 -- on every tile,
|
||||||
|
not just the K-capped ones. w2 (down) already uses warp_n=32.
|
||||||
|
"""
|
||||||
|
logger.info_once("Attempting to override humming GEMM config")
|
||||||
|
for entry in tuning_config:
|
||||||
|
config = entry[2]
|
||||||
|
block_shape = config.get("block_shape")
|
||||||
|
if not (block_shape and len(block_shape) == 3):
|
||||||
|
continue
|
||||||
|
block_m, block_n, block_k = block_shape
|
||||||
|
|
||||||
|
logger.info_once(f"Overriding humming GEMM config. Previous config\n: {config}")
|
||||||
|
if block_k > max_k_block:
|
||||||
|
config["block_shape"] = [block_m, block_n, max_k_block]
|
||||||
|
|
||||||
|
warp_shape = config.get("warp_shape")
|
||||||
|
if warp_shape and warp_shape[1] < 32 and block_n % 32 == 0:
|
||||||
|
config["warp_shape"] = [warp_shape[0], 32, warp_shape[2]]
|
||||||
|
|
||||||
|
logger.info_once(f"Overridden humming GEMM config. Current config\n: {config}")
|
||||||
|
|
||||||
|
|
||||||
class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -170,6 +201,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
|||||||
gemm_type=self.humming_gemm_type(),
|
gemm_type=self.humming_gemm_type(),
|
||||||
)
|
)
|
||||||
self.compute_config_str = json.dumps(self.compute_config)
|
self.compute_config_str = json.dumps(self.compute_config)
|
||||||
|
# Fix up the heuristic-chosen tiles before freezing to JSON -- see
|
||||||
|
# _fixup_moe_tuning_config.
|
||||||
|
_fixup_moe_tuning_config(self.w13_tuning_config)
|
||||||
|
_fixup_moe_tuning_config(self.w2_tuning_config)
|
||||||
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
|
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
|
||||||
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
|
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
|
||||||
|
|
||||||
@@ -178,12 +213,17 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
|||||||
sublayer_name: str,
|
sublayer_name: str,
|
||||||
inputs: torch.Tensor,
|
inputs: torch.Tensor,
|
||||||
quanted_input: torch.Tensor | None,
|
quanted_input: torch.Tensor | None,
|
||||||
|
input_scale: torch.Tensor | None = None,
|
||||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||||
from vllm.utils.humming import may_quant_input
|
from vllm.utils.humming import may_quant_input
|
||||||
|
|
||||||
|
# input_scale is set for block-FP8 (group-128) activations that were
|
||||||
|
# quantized before the EP dispatch: may_quant_input then skips the
|
||||||
|
# redundant w13 quantization and forwards the pre-computed scale.
|
||||||
return may_quant_input(
|
return may_quant_input(
|
||||||
self.humming_configs[sublayer_name],
|
self.humming_configs[sublayer_name],
|
||||||
inputs=inputs,
|
inputs=inputs,
|
||||||
|
input_scale=input_scale,
|
||||||
quanted_input=quanted_input,
|
quanted_input=quanted_input,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -263,6 +303,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
|||||||
(kMxfp4Static, kMxfp4Dynamic),
|
(kMxfp4Static, kMxfp4Dynamic),
|
||||||
(kMxfp4Static, kMxfp8Dynamic),
|
(kMxfp4Static, kMxfp8Dynamic),
|
||||||
(kMxfp4Static, kFp8DynamicTokenSym),
|
(kMxfp4Static, kFp8DynamicTokenSym),
|
||||||
|
# MXFP4 weight (group-32 e8m0) with block-FP8 activation
|
||||||
|
# (group-128 float32). Runs via WGMMA software dequant, so it
|
||||||
|
# works on Hopper (SM90/H200) as well as Blackwell.
|
||||||
|
(kMxfp4Static, kFp8Dynamic128Sym),
|
||||||
(kNvfp4Static, None),
|
(kNvfp4Static, None),
|
||||||
(kNvfp4Static, kFp8DynamicTokenSym),
|
(kNvfp4Static, kFp8DynamicTokenSym),
|
||||||
(kMxfp8Static, None),
|
(kMxfp8Static, None),
|
||||||
@@ -299,21 +343,43 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
|||||||
and _is_supported_wna16_weight_key(weight_key)
|
and _is_supported_wna16_weight_key(weight_key)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _prequantizes_dispatch_activation(self) -> bool:
|
||||||
|
"""
|
||||||
|
Whether the prepare/finalize step should quantize activations before
|
||||||
|
the (EP all-to-all) dispatch instead of leaving it to Humming.
|
||||||
|
|
||||||
|
This is enabled only for block-FP8 (group-128) activations: quantizing
|
||||||
|
to FP8 before dispatch sends FP8 rather than BF16 over the interconnect,
|
||||||
|
and Humming then consumes the pre-quantized FP8 + scale directly (see
|
||||||
|
the apply() methods, which forward the dispatch scale into
|
||||||
|
HummingMethod.may_quant_input, and may_quant_input itself, which is a
|
||||||
|
no-op when an input scale is already supplied). The scale layout
|
||||||
|
produced by vLLM's block-FP8 quantization ([M, K // 128] float32,
|
||||||
|
row-major) matches what the Humming WGMMA grouped GEMM expects.
|
||||||
|
"""
|
||||||
|
quant_config = self.quant_config
|
||||||
|
return (
|
||||||
|
quant_config.is_block_quantized
|
||||||
|
and quant_config.quant_dtype == current_platform.fp8_dtype()
|
||||||
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def expects_unquantized_inputs(self) -> bool:
|
def expects_unquantized_inputs(self) -> bool:
|
||||||
"""
|
"""
|
||||||
Humming kernels handle input quantization internally in apply().
|
Whether the prepare/finalize step should defer input quantization to
|
||||||
|
the experts (by setting defer_input_quant=True and passing unquantized
|
||||||
|
inputs).
|
||||||
|
|
||||||
This property tells the prepare/finalize step to skip input
|
Humming normally quantizes inputs internally via
|
||||||
quantization (by setting defer_input_quant=True) and pass
|
HummingMethod.may_quant_input() in apply(), so we defer quantization
|
||||||
unquantized inputs to the experts. This prevents double
|
(return True) to avoid quantizing twice -- once in prepare and once in
|
||||||
quantization: once in prepare and once in Humming's apply().
|
Humming's apply().
|
||||||
|
|
||||||
Returns:
|
The exception is block-FP8 (group-128) activations, which are quantized
|
||||||
True to indicate that this expert expects unquantized inputs
|
before the dispatch to save interconnect bandwidth (see
|
||||||
and will handle quantization internally.
|
_prequantizes_dispatch_activation): for those we must NOT defer.
|
||||||
"""
|
"""
|
||||||
return True
|
return not self._prequantizes_dispatch_activation()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _supports_current_device() -> bool:
|
def _supports_current_device() -> bool:
|
||||||
@@ -585,6 +651,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
|||||||
activation: MoEActivation,
|
activation: MoEActivation,
|
||||||
output: torch.Tensor,
|
output: torch.Tensor,
|
||||||
input: torch.Tensor,
|
input: torch.Tensor,
|
||||||
|
valid_rows: torch.Tensor | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
activation_config = self.activation_config
|
activation_config = self.activation_config
|
||||||
if (
|
if (
|
||||||
@@ -601,8 +668,84 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
|||||||
activation=activation,
|
activation=activation,
|
||||||
input=input,
|
input=input,
|
||||||
output=output,
|
output=output,
|
||||||
|
valid_rows=valid_rows,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool:
|
||||||
|
"""Whether the SITU activation + w2 quant can be fused into one kernel.
|
||||||
|
|
||||||
|
Fused only for k-major block-FP8 group-128 e4m3 with float32 scales --
|
||||||
|
the sole layout situ_and_mul_quant supports. A float32 as_dtype rules out
|
||||||
|
MXMMA (which uses e8m0 m-major scales), so a group-128 scale is
|
||||||
|
guaranteed k-major. Everything else (m-major MXFP8, other group sizes,
|
||||||
|
16-bit passthrough, non-SITU) falls back to the separate
|
||||||
|
situ_and_mul + quantize_input path.
|
||||||
|
"""
|
||||||
|
if activation != MoEActivation.SITU:
|
||||||
|
return False
|
||||||
|
w2cfg = self.humming_configs["w2"]
|
||||||
|
# Report the specific blocker once so a silent fallback to the unfused
|
||||||
|
# situ_and_mul + quant pair is never a mystery in a trace.
|
||||||
|
reason: str | None = None
|
||||||
|
if not (w2cfg.a_dtype.num_bits == 8 and str(w2cfg.a_dtype) == "float8e4m3"):
|
||||||
|
reason = f"w2 a_dtype is {w2cfg.a_dtype} (need float8e4m3)"
|
||||||
|
elif w2cfg.input_scale_group_size != 128:
|
||||||
|
reason = (
|
||||||
|
f"w2 input_scale_group_size is {w2cfg.input_scale_group_size} "
|
||||||
|
"(need 128)"
|
||||||
|
)
|
||||||
|
elif not (w2cfg.as_dtype is not None and str(w2cfg.as_dtype) == "float32"):
|
||||||
|
reason = f"w2 as_dtype is {w2cfg.as_dtype} (need float32, k-major)"
|
||||||
|
if reason is not None:
|
||||||
|
logger.warning_once(
|
||||||
|
"Humming fused SITU+FP8 quant disabled, using unfused "
|
||||||
|
"situ_and_mul + quant: %s",
|
||||||
|
reason,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def fused_situ_quant(
|
||||||
|
self,
|
||||||
|
gate_up_output: torch.Tensor,
|
||||||
|
quanted_down_input: torch.Tensor,
|
||||||
|
valid_rows: torch.Tensor | None,
|
||||||
|
topk: int = 1,
|
||||||
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
"""Run the fused SITU activation + FP8 quant for the w2 input.
|
||||||
|
|
||||||
|
Returns ``(quanted_down_input, input_scale)`` in the same layout the
|
||||||
|
unfused ``apply_activation`` + ``quantize_input("w2")`` pair produces: an
|
||||||
|
fp8 [M, d] tensor plus a k-major block-FP8 float32 [M, d // 128] group
|
||||||
|
scale. Only reached when ``fused_situ_quant_enabled`` (group_size 128).
|
||||||
|
"""
|
||||||
|
from vllm.model_executor.layers.fused_moe.activation import (
|
||||||
|
situ_and_mul_quant,
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg = self.activation_config
|
||||||
|
assert cfg.activation_situ_beta is not None, (
|
||||||
|
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||||
|
)
|
||||||
|
group_size = self.humming_configs["w2"].input_scale_group_size
|
||||||
|
m, d = quanted_down_input.size(0), quanted_down_input.size(1)
|
||||||
|
input_scale = torch.empty(
|
||||||
|
(m, d // group_size),
|
||||||
|
dtype=torch.float32,
|
||||||
|
device=quanted_down_input.device,
|
||||||
|
)
|
||||||
|
situ_and_mul_quant(
|
||||||
|
quanted_down_input,
|
||||||
|
input_scale,
|
||||||
|
gate_up_output,
|
||||||
|
beta=cfg.activation_situ_beta,
|
||||||
|
linear_beta=cfg.activation_situ_linear_beta,
|
||||||
|
group_size=group_size,
|
||||||
|
valid_rows=valid_rows,
|
||||||
|
topk=topk,
|
||||||
|
)
|
||||||
|
return quanted_down_input, input_scale
|
||||||
|
|
||||||
|
|
||||||
class HummingIndexedExperts(HummingExpertsBase):
|
class HummingIndexedExperts(HummingExpertsBase):
|
||||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||||
@@ -684,10 +827,12 @@ class HummingIndexedExperts(HummingExpertsBase):
|
|||||||
"""
|
"""
|
||||||
Standard apply implementation for Humming indexed experts.
|
Standard apply implementation for Humming indexed experts.
|
||||||
|
|
||||||
Humming performs activation quantization internally and consumes the
|
Note: Humming kernels handle weights internally through the layer
|
||||||
weights supplied by the modular kernel interface.
|
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
|
||||||
Results are written to the output buffer supplied by the modular
|
path (Humming quantizes the w13 input itself); for block-FP8 activations
|
||||||
kernel, which may alias workspace13.
|
it carries the scale computed before dispatch, which is forwarded to
|
||||||
|
may_quant_input so Humming skips the redundant w13 quantization. The
|
||||||
|
output is written into workspace13 via the buffer management.
|
||||||
"""
|
"""
|
||||||
assert not apply_router_weight_on_input
|
assert not apply_router_weight_on_input
|
||||||
|
|
||||||
@@ -710,6 +855,7 @@ class HummingIndexedExperts(HummingExpertsBase):
|
|||||||
inputs, input_scale = self.quantize_input(
|
inputs, input_scale = self.quantize_input(
|
||||||
"w13",
|
"w13",
|
||||||
inputs=hidden_states,
|
inputs=hidden_states,
|
||||||
|
input_scale=a1q_scale,
|
||||||
quanted_input=buffers.get("quanted_gate_up_input", None),
|
quanted_input=buffers.get("quanted_gate_up_input", None),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -722,17 +868,45 @@ class HummingIndexedExperts(HummingExpertsBase):
|
|||||||
**moe_kwargs1,
|
**moe_kwargs1,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.apply_activation(
|
# psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view.
|
||||||
activation=activation,
|
# situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds
|
||||||
input=buffers["gate_up_output"],
|
# on tokens, so both read this pointer directly -- no host cast/multiply.
|
||||||
output=buffers["activation_output"],
|
valid_tokens = None
|
||||||
)
|
topk = topk_ids.size(1)
|
||||||
|
if (
|
||||||
|
expert_tokens_meta is not None
|
||||||
|
and expert_tokens_meta.psum_recv_per_rank is not None
|
||||||
|
):
|
||||||
|
valid_tokens = expert_tokens_meta.psum_recv_per_rank[-1:]
|
||||||
|
|
||||||
inputs, input_scale = self.quantize_input(
|
if self.fused_situ_quant_enabled(activation):
|
||||||
"w2",
|
# Fused SITU + FP8 quant (per-token or block-FP8 group-128) straight
|
||||||
inputs=buffers["activation_output"],
|
# into the w2 input, skipping the bf16 activation_output round-trip.
|
||||||
quanted_input=buffers.get("quanted_down_input", None),
|
inputs, input_scale = self.fused_situ_quant(
|
||||||
)
|
gate_up_output=buffers["gate_up_output"],
|
||||||
|
quanted_down_input=buffers["quanted_down_input"],
|
||||||
|
valid_rows=valid_tokens,
|
||||||
|
topk=topk,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Fallback situ_and_mul takes int64 row counts (tokens*topk).
|
||||||
|
valid_rows = (
|
||||||
|
valid_tokens.to(torch.int64) * topk
|
||||||
|
if valid_tokens is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
self.apply_activation(
|
||||||
|
activation=activation,
|
||||||
|
input=buffers["gate_up_output"],
|
||||||
|
output=buffers["activation_output"],
|
||||||
|
valid_rows=valid_rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
inputs, input_scale = self.quantize_input(
|
||||||
|
"w2",
|
||||||
|
inputs=buffers["activation_output"],
|
||||||
|
quanted_input=buffers.get("quanted_down_input", None),
|
||||||
|
)
|
||||||
|
|
||||||
self.humming_forward(
|
self.humming_forward(
|
||||||
"w2",
|
"w2",
|
||||||
@@ -743,12 +917,16 @@ class HummingIndexedExperts(HummingExpertsBase):
|
|||||||
**moe_kwargs2,
|
**moe_kwargs2,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# expert_map masks any non-local id; num_valid_tokens bounds the
|
||||||
|
# persistent kernel to the real token rows [0, num_recv) so the padding
|
||||||
|
# tail is never iterated (CUDA-graph-safe device scalar).
|
||||||
moe_fused_mul_sum(
|
moe_fused_mul_sum(
|
||||||
inputs=buffers["down_output"].view(*topk_ids.shape, -1),
|
inputs=buffers["down_output"].view(*topk_ids.shape, -1),
|
||||||
topk_weights=topk_weights,
|
topk_weights=topk_weights,
|
||||||
topk_ids=topk_ids,
|
topk_ids=topk_ids,
|
||||||
expert_map=expert_map,
|
expert_map=expert_map,
|
||||||
outputs=output,
|
outputs=output,
|
||||||
|
num_valid_tokens=valid_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -787,10 +965,13 @@ class HummingGroupedExperts(HummingExpertsBase):
|
|||||||
"""
|
"""
|
||||||
Standard apply implementation for Humming grouped experts.
|
Standard apply implementation for Humming grouped experts.
|
||||||
|
|
||||||
Humming performs activation quantization internally and consumes the
|
Note: Humming kernels handle weights internally through the layer
|
||||||
weights supplied by the modular kernel interface.
|
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
|
||||||
Results are written to the output buffer supplied by the modular
|
path (Humming quantizes the w13 input itself); for block-FP8 activations
|
||||||
kernel, which may alias workspace13.
|
it carries the scale computed before dispatch. It is permuted alongside
|
||||||
|
the tokens by moe_permute and forwarded to may_quant_input so Humming
|
||||||
|
skips the redundant w13 quantization. The output is written into
|
||||||
|
workspace13 via the buffer management.
|
||||||
"""
|
"""
|
||||||
assert not apply_router_weight_on_input
|
assert not apply_router_weight_on_input
|
||||||
|
|
||||||
@@ -805,9 +986,9 @@ class HummingGroupedExperts(HummingExpertsBase):
|
|||||||
)
|
)
|
||||||
buffers["output"] = output
|
buffers["output"] = output
|
||||||
|
|
||||||
hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute(
|
hidden_states, a1q_scale, expert_first_token_offset, inv_perm, _ = moe_permute(
|
||||||
hidden_states=hidden_states,
|
hidden_states=hidden_states,
|
||||||
a1q_scale=None,
|
a1q_scale=a1q_scale,
|
||||||
topk_ids=topk_ids,
|
topk_ids=topk_ids,
|
||||||
n_expert=global_num_experts,
|
n_expert=global_num_experts,
|
||||||
n_local_expert=self.num_experts,
|
n_local_expert=self.num_experts,
|
||||||
@@ -818,6 +999,7 @@ class HummingGroupedExperts(HummingExpertsBase):
|
|||||||
inputs, input_scale = self.quantize_input(
|
inputs, input_scale = self.quantize_input(
|
||||||
"w13",
|
"w13",
|
||||||
inputs=hidden_states,
|
inputs=hidden_states,
|
||||||
|
input_scale=a1q_scale,
|
||||||
quanted_input=buffers.get("quanted_gate_up_input", None),
|
quanted_input=buffers.get("quanted_gate_up_input", None),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -901,15 +1083,21 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
|||||||
"""
|
"""
|
||||||
Standard apply implementation for Humming batched grouped experts.
|
Standard apply implementation for Humming batched grouped experts.
|
||||||
|
|
||||||
Humming performs activation quantization internally and consumes the
|
Note: Humming kernels handle weights internally through the layer
|
||||||
weights supplied by the modular kernel interface.
|
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
|
||||||
Results are written to the output buffer supplied by the modular
|
path (Humming quantizes the w13 input itself); for block-FP8 activations
|
||||||
kernel, which may alias workspace13.
|
it carries the scale computed before dispatch, which is forwarded to
|
||||||
|
may_quant_input so Humming skips the redundant w13 quantization. The
|
||||||
|
output is written into workspace13 via the buffer management.
|
||||||
"""
|
"""
|
||||||
assert not apply_router_weight_on_input
|
assert not apply_router_weight_on_input
|
||||||
assert expert_tokens_meta is not None
|
assert expert_tokens_meta is not None
|
||||||
|
|
||||||
hidden_states = hidden_states.view(-1, hidden_states.size(-1))
|
hidden_states = hidden_states.view(-1, hidden_states.size(-1))
|
||||||
|
# Keep the (batched) block-FP8 scale row-aligned with the flattened
|
||||||
|
# [num_experts * max_tokens, K] hidden states above.
|
||||||
|
if a1q_scale is not None and a1q_scale.dim() == 3:
|
||||||
|
a1q_scale = a1q_scale.view(-1, a1q_scale.size(-1))
|
||||||
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
|
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
|
||||||
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
||||||
|
|
||||||
@@ -925,6 +1113,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
|||||||
inputs, input_scale = self.quantize_input(
|
inputs, input_scale = self.quantize_input(
|
||||||
"w13",
|
"w13",
|
||||||
inputs=hidden_states,
|
inputs=hidden_states,
|
||||||
|
input_scale=a1q_scale,
|
||||||
quanted_input=buffers.get("quanted_gate_up_input", None),
|
quanted_input=buffers.get("quanted_gate_up_input", None),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -100,8 +100,9 @@ class ExpertTokensMetadata:
|
|||||||
Metadata regarding expert-token routing.
|
Metadata regarding expert-token routing.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
expert_num_tokens: torch.Tensor
|
expert_num_tokens: torch.Tensor | None
|
||||||
expert_num_tokens_cpu: torch.Tensor | None
|
expert_num_tokens_cpu: torch.Tensor | None
|
||||||
|
psum_recv_per_rank: torch.Tensor | None = None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def make_from_list(
|
def make_from_list(
|
||||||
@@ -901,6 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
|||||||
*,
|
*,
|
||||||
topk_ids: torch.Tensor | None = None,
|
topk_ids: torch.Tensor | None = None,
|
||||||
expert_map: torch.Tensor | None = None,
|
expert_map: torch.Tensor | None = None,
|
||||||
|
valid_rows: torch.Tensor | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
apply_moe_activation(
|
apply_moe_activation(
|
||||||
activation,
|
activation,
|
||||||
@@ -909,6 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
|||||||
activation_config=self.activation_config,
|
activation_config=self.activation_config,
|
||||||
topk_ids=topk_ids,
|
topk_ids=topk_ids,
|
||||||
expert_map=expert_map,
|
expert_map=expert_map,
|
||||||
|
valid_rows=valid_rows,
|
||||||
)
|
)
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
import torch
|
import torch
|
||||||
from torch._subclasses.fake_tensor import FakeTensor
|
from torch._subclasses.fake_tensor import FakeTensor
|
||||||
|
|
||||||
from vllm.platforms import current_platform
|
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
|
|
||||||
|
|
||||||
@@ -14,122 +13,75 @@ def moe_fused_mul_sum_kernel(
|
|||||||
outputs_ptr,
|
outputs_ptr,
|
||||||
top_ids_ptr,
|
top_ids_ptr,
|
||||||
expert_map_ptr,
|
expert_map_ptr,
|
||||||
num_tokens,
|
num_valid_tokens_ptr,
|
||||||
stride_m,
|
stride_m,
|
||||||
|
has_topk_ids: tl.constexpr,
|
||||||
has_expert_map: tl.constexpr,
|
has_expert_map: tl.constexpr,
|
||||||
|
has_num_valid: tl.constexpr,
|
||||||
top_k: tl.constexpr,
|
top_k: tl.constexpr,
|
||||||
size: tl.constexpr,
|
hidden_size: tl.constexpr,
|
||||||
BLOCK_M: tl.constexpr,
|
|
||||||
BLOCK_K: tl.constexpr,
|
BLOCK_K: tl.constexpr,
|
||||||
):
|
):
|
||||||
pid_k = tl.program_id(0)
|
# One CTA owns one token's output row. Resolve which slots to sum once per
|
||||||
pid_m = tl.program_id(1)
|
# row (the decision is invariant over hidden tiles); the `if take` branch is
|
||||||
|
# block-uniform, so padding slots are skipped rather than masked-and-loaded.
|
||||||
|
pid_m = tl.program_id(0)
|
||||||
|
# Bound to the real recv rows [0, num_recv). Under cudagraph decode the grid
|
||||||
|
# is the static padded token count, and padding rows past num_recv carry
|
||||||
|
# stale ids and stale NaN in `inputs`; returning before any load leaves their
|
||||||
|
# output rows untouched instead of summing that garbage. The constexpr guard
|
||||||
|
# short-circuits, so num_valid_tokens_ptr is only read when it is provided.
|
||||||
|
if has_num_valid and pid_m >= tl.load(num_valid_tokens_ptr).to(tl.int32):
|
||||||
|
return
|
||||||
|
w_row = topk_weights_ptr + pid_m * top_k
|
||||||
|
|
||||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
takes = ()
|
||||||
offs_k = pid_k * BLOCK_K + tl.arange(0, BLOCK_K)
|
weights = ()
|
||||||
|
if has_topk_ids:
|
||||||
|
any_valid = 0
|
||||||
|
for n in tl.static_range(top_k):
|
||||||
|
id_val = tl.load(top_ids_ptr + pid_m * top_k + n)
|
||||||
|
valid = id_val >= 0
|
||||||
|
any_valid += valid.to(tl.int32)
|
||||||
|
take = valid
|
||||||
|
if has_expert_map:
|
||||||
|
local_id = tl.load(expert_map_ptr + tl.where(valid, id_val, 0))
|
||||||
|
take = valid & (local_id >= 0)
|
||||||
|
takes += (take,) # type: ignore[assignment]
|
||||||
|
weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment]
|
||||||
|
# All-padding rows early-return untouched (decode contract past num_recv).
|
||||||
|
if any_valid == 0:
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
for n in tl.static_range(top_k):
|
||||||
|
takes += (True,) # type: ignore[assignment]
|
||||||
|
weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment]
|
||||||
|
|
||||||
m_mask = offs_m < num_tokens
|
n_tiles: tl.constexpr = (hidden_size + BLOCK_K - 1) // BLOCK_K
|
||||||
k_mask = offs_k < size
|
a_row = inputs_ptr + pid_m * stride_m
|
||||||
mask = m_mask[:, None] & k_mask[None, :]
|
out_row = outputs_ptr + pid_m * hidden_size
|
||||||
|
|
||||||
a_base = inputs_ptr + (offs_m * stride_m)[:, None] + offs_k[None, :]
|
for t in tl.range(0, n_tiles):
|
||||||
b_base = topk_weights_ptr + offs_m * top_k
|
offs_k = t * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||||
|
kmask = offs_k < hidden_size
|
||||||
acc = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32)
|
acc = tl.zeros((BLOCK_K,), dtype=tl.float32)
|
||||||
|
for n in tl.static_range(top_k):
|
||||||
for n in tl.static_range(top_k):
|
if takes[n]:
|
||||||
b_val = tl.load(b_base + n, mask=m_mask, other=0.0).to(tl.float32)
|
a = tl.load(a_row + n * hidden_size + offs_k, mask=kmask, other=0.0)
|
||||||
if has_expert_map:
|
acc += a.to(tl.float32) * weights[n]
|
||||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
|
tl.store(out_row + offs_k, acc.to(outputs_ptr.dtype.element_ty), mask=kmask)
|
||||||
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
|
|
||||||
a_vec = tl.load(
|
|
||||||
a_base + n * size,
|
|
||||||
mask=mask & expert_mask[:, None],
|
|
||||||
other=0.0,
|
|
||||||
).to(tl.float32)
|
|
||||||
else:
|
|
||||||
a_vec = tl.load(
|
|
||||||
a_base + n * size,
|
|
||||||
mask=mask,
|
|
||||||
other=0.0,
|
|
||||||
).to(tl.float32)
|
|
||||||
acc += a_vec * b_val[:, None]
|
|
||||||
|
|
||||||
out_ptrs = outputs_ptr + (offs_m * size)[:, None] + offs_k[None, :]
|
|
||||||
tl.store(
|
|
||||||
out_ptrs,
|
|
||||||
acc.to(outputs_ptr.dtype.element_ty),
|
|
||||||
mask=mask,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _heuristic_config(
|
def _heuristic_config(
|
||||||
num_tokens: int,
|
hidden_size: int,
|
||||||
top_k: int,
|
|
||||||
size: int,
|
|
||||||
element_size: int,
|
element_size: int,
|
||||||
):
|
):
|
||||||
is_fp32 = element_size > 2
|
is_fp32 = element_size > 2
|
||||||
is_sm90_plus = current_platform.has_device_capability(90)
|
max_block_k = 256 if is_fp32 else 512
|
||||||
is_sm80_before = not current_platform.has_device_capability(80)
|
BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
|
||||||
|
num_warps = 4 if is_fp32 else 2
|
||||||
if current_platform.has_device_capability(90):
|
num_stages = 3
|
||||||
# SM90/SM100+: prefer small tiles + many CTAs.
|
return BLOCK_K, num_warps, num_stages
|
||||||
if is_fp32:
|
|
||||||
BLOCK_M = 1 if num_tokens <= 4 else 2
|
|
||||||
else:
|
|
||||||
if num_tokens <= 4:
|
|
||||||
BLOCK_M = 1
|
|
||||||
elif num_tokens <= 128:
|
|
||||||
BLOCK_M = 2
|
|
||||||
else:
|
|
||||||
BLOCK_M = 4
|
|
||||||
elif is_fp32:
|
|
||||||
if num_tokens <= 4:
|
|
||||||
BLOCK_M = 1
|
|
||||||
elif num_tokens <= 32:
|
|
||||||
BLOCK_M = 2
|
|
||||||
elif num_tokens <= 128:
|
|
||||||
BLOCK_M = 4
|
|
||||||
else:
|
|
||||||
BLOCK_M = 4
|
|
||||||
else:
|
|
||||||
if num_tokens <= 4:
|
|
||||||
BLOCK_M = 1
|
|
||||||
elif num_tokens <= 32:
|
|
||||||
BLOCK_M = 2
|
|
||||||
elif num_tokens <= 128:
|
|
||||||
BLOCK_M = 4
|
|
||||||
elif num_tokens <= 1024:
|
|
||||||
BLOCK_M = 16
|
|
||||||
else:
|
|
||||||
BLOCK_M = 8
|
|
||||||
|
|
||||||
if is_fp32:
|
|
||||||
max_block_k = 256
|
|
||||||
elif is_sm80_before or is_sm90_plus:
|
|
||||||
max_block_k = 512
|
|
||||||
else:
|
|
||||||
max_block_k = 1024
|
|
||||||
BLOCK_K = min(triton.next_power_of_2(size), max_block_k)
|
|
||||||
BLOCK_K = max(BLOCK_K, 256)
|
|
||||||
|
|
||||||
total = BLOCK_M * BLOCK_K
|
|
||||||
if is_fp32:
|
|
||||||
num_warps = max(8, min(16, total // 64))
|
|
||||||
else:
|
|
||||||
num_warps = max(4, min(16, total // 256))
|
|
||||||
|
|
||||||
if is_sm80_before:
|
|
||||||
num_warps = min(num_warps, 8)
|
|
||||||
num_stages = 2
|
|
||||||
elif is_sm90_plus:
|
|
||||||
num_warps = min(num_warps, 8)
|
|
||||||
num_stages = 4 if total <= 2048 else 2
|
|
||||||
else:
|
|
||||||
num_stages = 4 if total <= 2048 else 2
|
|
||||||
|
|
||||||
return BLOCK_M, BLOCK_K, num_warps, num_stages
|
|
||||||
|
|
||||||
|
|
||||||
def moe_fused_mul_sum(
|
def moe_fused_mul_sum(
|
||||||
@@ -138,6 +90,7 @@ def moe_fused_mul_sum(
|
|||||||
outputs: torch.Tensor | None = None,
|
outputs: torch.Tensor | None = None,
|
||||||
topk_ids: torch.Tensor | None = None,
|
topk_ids: torch.Tensor | None = None,
|
||||||
expert_map: torch.Tensor | None = None,
|
expert_map: torch.Tensor | None = None,
|
||||||
|
num_valid_tokens: torch.Tensor | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
Fused kernel for MoE (Mixture of Experts) to perform weighted summation
|
Fused kernel for MoE (Mixture of Experts) to perform weighted summation
|
||||||
@@ -150,10 +103,21 @@ def moe_fused_mul_sum(
|
|||||||
Shape: (num_tokens, top_k).
|
Shape: (num_tokens, top_k).
|
||||||
outputs: Optional pre-allocated output tensor.
|
outputs: Optional pre-allocated output tensor.
|
||||||
Shape: (num_tokens, hidden_size).
|
Shape: (num_tokens, hidden_size).
|
||||||
topk_ids: Optional indices of the top-k experts. Used when
|
topk_ids: Optional indices of the top-k experts. Shape:
|
||||||
`expert_map` is provided. Shape: (num_tokens, top_k).
|
(num_tokens, top_k). A value of -1 marks a slot the expert GEMM
|
||||||
|
skipped; those slots are excluded from the sum. When provided, rows
|
||||||
|
with all top ids < 0 (worst-case padding) are skipped and their
|
||||||
|
output rows left untouched. Required when `expert_map` is provided.
|
||||||
expert_map: Optional mapping for Expert Parallelism. A value < 0
|
expert_map: Optional mapping for Expert Parallelism. A value < 0
|
||||||
indicates an invalid token/expert pair that will be skipped.
|
indicates an invalid token/expert pair that will be skipped. Only
|
||||||
|
needed when `topk_ids` may contain non-local expert ids; if every
|
||||||
|
non-(-1) id is already a local expert, leave it None to skip the
|
||||||
|
redundant per-slot lookup.
|
||||||
|
num_valid_tokens: Optional device scalar (1-element tensor) holding the
|
||||||
|
number of real token rows (num_recv for a decode dispatch). When
|
||||||
|
provided, rows past it are left untouched, so the static cudagraph
|
||||||
|
grid never sums stale padding rows. Pass the token count, not
|
||||||
|
token*top_k.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The fused weighted sum of expert outputs.
|
The fused weighted sum of expert outputs.
|
||||||
@@ -166,34 +130,40 @@ def moe_fused_mul_sum(
|
|||||||
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||||
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||||
|
|
||||||
num_tokens, top_k, size = inputs.shape
|
num_tokens, top_k, hidden_size = inputs.shape
|
||||||
output_shape = (num_tokens, size)
|
output_shape = (num_tokens, hidden_size)
|
||||||
if outputs is None:
|
if outputs is None:
|
||||||
outputs = torch.empty(output_shape, dtype=inputs.dtype, device=inputs.device)
|
outputs = torch.empty(output_shape, dtype=inputs.dtype, device=inputs.device)
|
||||||
|
|
||||||
assert outputs.shape == output_shape
|
assert outputs.shape == output_shape
|
||||||
assert topk_weights.shape == (num_tokens, top_k)
|
assert topk_weights.shape == (num_tokens, top_k)
|
||||||
|
assert expert_map is None or topk_ids is not None, (
|
||||||
|
"topk_ids is required to interpret expert_map"
|
||||||
|
)
|
||||||
|
if topk_ids is not None:
|
||||||
|
assert topk_ids.shape == (num_tokens, top_k)
|
||||||
|
assert topk_ids.is_contiguous()
|
||||||
|
assert topk_ids.dtype in (torch.int32, torch.int64)
|
||||||
|
|
||||||
if not isinstance(inputs, FakeTensor):
|
if not isinstance(inputs, FakeTensor):
|
||||||
BLOCK_M, BLOCK_K, num_warps, num_stages = _heuristic_config(
|
BLOCK_K, num_warps, num_stages = _heuristic_config(
|
||||||
num_tokens,
|
hidden_size,
|
||||||
top_k,
|
|
||||||
size,
|
|
||||||
inputs.element_size(),
|
inputs.element_size(),
|
||||||
)
|
)
|
||||||
grid = (triton.cdiv(size, BLOCK_K), triton.cdiv(num_tokens, BLOCK_M))
|
grid = (num_tokens,)
|
||||||
moe_fused_mul_sum_kernel[grid](
|
moe_fused_mul_sum_kernel[grid](
|
||||||
inputs,
|
inputs,
|
||||||
topk_weights,
|
topk_weights,
|
||||||
outputs,
|
outputs,
|
||||||
topk_ids,
|
topk_ids,
|
||||||
expert_map,
|
expert_map,
|
||||||
num_tokens,
|
num_valid_tokens,
|
||||||
top_k * size,
|
top_k * hidden_size,
|
||||||
|
topk_ids is not None,
|
||||||
expert_map is not None,
|
expert_map is not None,
|
||||||
|
num_valid_tokens is not None,
|
||||||
top_k,
|
top_k,
|
||||||
size,
|
hidden_size,
|
||||||
BLOCK_M,
|
|
||||||
BLOCK_K,
|
BLOCK_K,
|
||||||
num_warps=num_warps,
|
num_warps=num_warps,
|
||||||
num_stages=num_stages,
|
num_stages=num_stages,
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
|||||||
- Worst-case tensor allocation; padding rows zeroed via
|
- Worst-case tensor allocation; padding rows zeroed via
|
||||||
handle.psum_num_recv_tokens_per_scaleup_rank
|
handle.psum_num_recv_tokens_per_scaleup_rank
|
||||||
- Fully cudagraph-capturable
|
- Fully cudagraph-capturable
|
||||||
- Expert kernel sorts internally (expert_tokens_meta=None)
|
- Expert kernel sorts internally (expert_tokens_meta carries no counts)
|
||||||
|
|
||||||
**Prefill mode (use_cudagraph=False):**
|
**Prefill mode (use_cudagraph=False):**
|
||||||
- do_expand=True, do_cpu_sync=True
|
- do_expand=True, do_cpu_sync=True
|
||||||
@@ -266,6 +266,18 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
|||||||
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
|
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
|
||||||
recv_topk_weights = recv_topk_weights.unsqueeze(1)
|
recv_topk_weights = recv_topk_weights.unsqueeze(1)
|
||||||
|
|
||||||
|
if self.use_cudagraph:
|
||||||
|
# Carry the per-rank prefix sum so SiTU can skip padding rows.
|
||||||
|
# expert_num_tokens stays None: count-based consumers (DeepGEMM,
|
||||||
|
# Triton) must treat a None field as "no counts" and derive their
|
||||||
|
# own, exactly as in the meta-absent decode case.
|
||||||
|
if expert_tokens_meta is None:
|
||||||
|
expert_tokens_meta = mk.ExpertTokensMetadata(
|
||||||
|
expert_num_tokens=None,
|
||||||
|
expert_num_tokens_cpu=None,
|
||||||
|
)
|
||||||
|
expert_tokens_meta.psum_recv_per_rank = psum_recv_per_rank
|
||||||
|
|
||||||
if not quant_config.is_block_quantized and not defer_input_quant:
|
if not quant_config.is_block_quantized and not defer_input_quant:
|
||||||
expert_x_scale = None
|
expert_x_scale = None
|
||||||
if expert_x.numel() != 0:
|
if expert_x.numel() != 0:
|
||||||
|
|||||||
@@ -307,7 +307,18 @@ def _humming_input_schema_to_quant_key(
|
|||||||
gs = schema.input_scale_group_size
|
gs = schema.input_scale_group_size
|
||||||
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
|
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
|
||||||
|
|
||||||
scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32
|
# Pick the scale dtype the Humming kernel actually consumes. An explicit
|
||||||
|
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
|
||||||
|
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
|
||||||
|
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
|
||||||
|
# float32 scale. Getting this right lets a grouped FP8 activation match
|
||||||
|
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
|
||||||
|
if schema.input_scale_dtype is not None:
|
||||||
|
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
|
||||||
|
elif gs == 32:
|
||||||
|
scale_dtype = MXFP_SCALE_DTYPE
|
||||||
|
else:
|
||||||
|
scale_dtype = torch.float32
|
||||||
|
|
||||||
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
|
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
|
||||||
|
|
||||||
@@ -553,6 +564,7 @@ def make_humming_moe_quant_config(
|
|||||||
quant_dtype: torch.dtype | str | None,
|
quant_dtype: torch.dtype | str | None,
|
||||||
weight_dtype: torch.dtype | str | None,
|
weight_dtype: torch.dtype | str | None,
|
||||||
weight_group_shape: GroupShape | None = None,
|
weight_group_shape: GroupShape | None = None,
|
||||||
|
activation_group_shape: GroupShape | None = None,
|
||||||
w1_scale: torch.Tensor | None = None,
|
w1_scale: torch.Tensor | None = None,
|
||||||
w2_scale: torch.Tensor | None = None,
|
w2_scale: torch.Tensor | None = None,
|
||||||
w1_zp: torch.Tensor | None = None,
|
w1_zp: torch.Tensor | None = None,
|
||||||
@@ -569,7 +581,14 @@ def make_humming_moe_quant_config(
|
|||||||
assert humming_configs is not None
|
assert humming_configs is not None
|
||||||
if quant_dtype is None:
|
if quant_dtype is None:
|
||||||
a_quant_desc = FusedMoEQuantDesc(dtype=None)
|
a_quant_desc = FusedMoEQuantDesc(dtype=None)
|
||||||
|
elif activation_group_shape is not None:
|
||||||
|
# Pre-dispatch quantization.
|
||||||
|
a_quant_desc = FusedMoEQuantDesc(
|
||||||
|
dtype=quant_dtype, shape=activation_group_shape
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
|
# Deferred path: Humming quantizes the activation internally, so the
|
||||||
|
# descriptor only needs a non-None dtype to mark it as quantized.
|
||||||
shape = GroupShape(row=1, col=-1)
|
shape = GroupShape(row=1, col=-1)
|
||||||
a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape)
|
a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape)
|
||||||
|
|
||||||
@@ -621,6 +640,19 @@ def get_humming_moe_quant_config(
|
|||||||
else:
|
else:
|
||||||
q_dtype = str(input_schema.a_dtype)
|
q_dtype = str(input_schema.a_dtype)
|
||||||
|
|
||||||
|
# Block-FP8 (group-128) activations are quantized *before* the EP all-to-all
|
||||||
|
# dispatch (so FP8 rather than BF16 crosses the interconnect) and consumed by
|
||||||
|
# Humming as-is.
|
||||||
|
activation_group_shape: GroupShape | None = None
|
||||||
|
input_scale_group_size = getattr(input_schema, "input_scale_group_size", 0) or 0
|
||||||
|
if (
|
||||||
|
q_dtype is not None
|
||||||
|
and q_dtype.startswith("float8")
|
||||||
|
and input_scale_group_size == 128
|
||||||
|
):
|
||||||
|
q_dtype = _HUMMING_TO_QUANT_DTYPE.get(input_schema.a_dtype, FP8_DTYPE)
|
||||||
|
activation_group_shape = GroupShape(row=1, col=input_scale_group_size)
|
||||||
|
|
||||||
weight_scale_group_size = weight_schema.weight_scale_group_size
|
weight_scale_group_size = weight_schema.weight_scale_group_size
|
||||||
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
|
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
|
||||||
weight_group_shape: tuple[int, ...] = ()
|
weight_group_shape: tuple[int, ...] = ()
|
||||||
@@ -638,6 +670,7 @@ def get_humming_moe_quant_config(
|
|||||||
quant_dtype=q_dtype,
|
quant_dtype=q_dtype,
|
||||||
weight_dtype=str(weight_schema.b_dtype),
|
weight_dtype=str(weight_schema.b_dtype),
|
||||||
weight_group_shape=weight_group_shape,
|
weight_group_shape=weight_group_shape,
|
||||||
|
activation_group_shape=activation_group_shape,
|
||||||
w1_scale=getattr(layer, "w13_weight_scale", None),
|
w1_scale=getattr(layer, "w13_weight_scale", None),
|
||||||
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
|
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
|
||||||
w1_zp=getattr(layer, "w13_zero_point", None),
|
w1_zp=getattr(layer, "w13_zero_point", None),
|
||||||
|
|||||||
Reference in New Issue
Block a user