[NVFP4][Emulation] Fuse NVFP4 weight dequantization with compute in triton kernel for w13/w2 MOE MLP linears (#44667)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-06-25 19:33:00 -07:00
committed by GitHub
parent 02a1f23711
commit 552a9dbe59
3 changed files with 905 additions and 67 deletions
@@ -11,6 +11,8 @@ Weights are dequantized on the fly during each forward, we fall back to calling
is applied on `a13`, `a2`.
"""
from typing import Any
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -21,18 +23,316 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
from vllm.model_executor.layers.fused_moe.fused_moe import (
try_get_optimal_moe_config,
write_zeros_to_output,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
moe_kernel_quantize_input,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_to_dtype,
_e2m1_inline,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.triton_utils import tl, triton
logger = init_logger(__name__)
@triton.jit
def fused_moe_nvfp4_emulation_kernel(
a_ptr,
b_ptr,
c_ptr,
b_scale_ptr,
w_global_scale_ptr,
topk_weights_ptr,
sorted_token_ids_ptr,
expert_ids_ptr,
num_tokens_post_padded_ptr,
N: tl.constexpr,
K: tl.constexpr,
EM,
num_valid_tokens,
# Strides — A [M, K]
stride_am,
stride_ak,
# Strides — B [E, N, K//2], passed as (expert, K-packed, N)
stride_be,
stride_bk,
stride_bn,
# Strides — C [M, topk, N]
stride_cm,
stride_cn,
# Strides — B_scale [E, N, K//BLOCK], passed as (expert, K-scale, N)
stride_bse,
stride_bsk,
stride_bsn,
block_k_diviable: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
GROUP_SIZE_M: tl.constexpr,
MUL_ROUTED_WEIGHT: tl.constexpr,
top_k: tl.constexpr,
compute_type: tl.constexpr,
group_size: tl.constexpr,
):
"""
Fused MoE kernel for emulated NVFP4 weight-only dequantization + GEMM.
Activations A are BF16 (already QDQ'd externally).
Weights B are packed uint8 NVFP4 [E, N, K//2] — two FP4 values per byte
along the K dimension.
B_scale holds per-block FP8-E4M3 scales [E, N, K // group_size].
w_global_scale is a per-expert scalar global scale.
The dequantization formula per element is:
w_float = e2m1_decode(nibble) * (block_scale_fp8 * global_scale)
Weight loading optimization: each packed byte is loaded exactly once as
a [BLOCK_SIZE_N, BLOCK_SIZE_K // 2] tile (N-major), both nibbles are
extracted, decoded and scaled, then tl.interleave produces the
[BLOCK_SIZE_N, BLOCK_SIZE_K] dequantized tile which is transposed to
[BLOCK_SIZE_K, BLOCK_SIZE_N] for tl.dot.
"""
BLOCK_SIZE_K_PACKED: tl.constexpr = BLOCK_SIZE_K // 2
# Map program ids to the block of C it should compute.
pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
num_pid_in_group = GROUP_SIZE_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
# Token / expert setup
num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr)
if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
return
offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64)
token_mask = offs_token < num_valid_tokens
off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
if off_experts == -1:
write_zeros_to_output(
c_ptr,
stride_cm,
stride_cn,
pid_n,
N,
offs_token,
token_mask,
BLOCK_SIZE_M,
BLOCK_SIZE_N,
compute_type,
)
return
# Pointer setup
offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
offs_k = tl.arange(0, BLOCK_SIZE_K)
offs_k_packed = tl.arange(0, BLOCK_SIZE_K_PACKED)
# A pointers: [BLOCK_SIZE_M, BLOCK_SIZE_K]
a_ptrs = a_ptr + (
offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak
)
# B pointers: [BLOCK_SIZE_N, BLOCK_SIZE_K_PACKED] — N-major so that
# tl.interleave (which operates on the last dim) produces a
# [BLOCK_SIZE_N, BLOCK_SIZE_K] tile that we transpose for tl.dot.
# Each unique byte is loaded exactly once.
b_ptrs = (
b_ptr
+ off_experts * stride_be
+ offs_bn[:, None] * stride_bn
+ offs_k_packed[None, :] * stride_bk
)
# B_scale pointers: [BLOCK_SIZE_N, BLOCK_SIZE_K_PACKED] — same
# N-major layout. Each packed byte index covers 2 K elements that
# always fall within the same group (group_size=16, so each group
# spans 8 packed bytes). We can therefore index the scale using
# offs_k_packed directly.
# Note: group_size_packed = group_size // 2 maps packed indices to
# scale indices the same way unpacked indices map via group_size.
group_size_packed: tl.constexpr = group_size // 2
# Load per-expert global scale (scalar).
w_global_scale = tl.load(w_global_scale_ptr + off_experts).to(tl.float32)
# K-loop with FP32 accumulation
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
# Load A tile [BLOCK_SIZE_M, BLOCK_SIZE_K].
if block_k_diviable:
a = tl.load(
a_ptrs,
mask=token_mask[:, None],
other=0.0,
)
else:
a = tl.load(
a_ptrs,
mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K),
other=0.0,
)
# Load packed weight tile [BLOCK_SIZE_N, BLOCK_SIZE_K_PACKED].
if block_k_diviable:
raw_bytes = tl.load(b_ptrs)
else:
kp_mask = offs_k_packed[None, :] < (K // 2) - k * BLOCK_SIZE_K_PACKED
raw_bytes = tl.load(b_ptrs, mask=kp_mask, other=0)
# Extract both nibbles from each byte (each [N, K_packed]).
low_nibble = raw_bytes & 0x0F
high_nibble = (raw_bytes >> 4) & 0x0F
low_decoded = _e2m1_inline(low_nibble)
high_decoded = _e2m1_inline(high_nibble)
# Load and apply per-block FP8 scales.
# Scale shape: [BLOCK_SIZE_N, BLOCK_SIZE_K_PACKED], one scale per
# group_size_packed packed elements.
b_scale_ptrs = (
b_scale_ptr
+ off_experts * stride_bse
+ offs_bn[:, None] * stride_bsn
+ ((offs_k_packed[None, :] + BLOCK_SIZE_K_PACKED * k) // group_size_packed)
* stride_bsk
)
if block_k_diviable:
b_scale_raw = tl.load(b_scale_ptrs)
else:
b_scale_raw = tl.load(b_scale_ptrs, mask=kp_mask, other=0.0)
b_scale = tl.cast(b_scale_raw, tl.float8e4nv, bitcast=True).to(tl.float32)
b_scale = b_scale * w_global_scale
# Scale both halves with the same per-block scale (the two
# elements packed in one byte always belong to the same group).
low_scaled = low_decoded * b_scale
high_scaled = high_decoded * b_scale
# Interleave along last dim: [N, K_packed] x2 -> [N, K],
# then transpose to [K, N] for tl.dot.
b = tl.trans(tl.interleave(low_scaled, high_scaled)).to(compute_type)
accumulator = tl.dot(a, b, acc=accumulator)
# Advance pointers along K.
a_ptrs += BLOCK_SIZE_K * stride_ak
b_ptrs += BLOCK_SIZE_K_PACKED * stride_bk
# Router weight multiplication (in float32 for stability)
if MUL_ROUTED_WEIGHT:
moe_weight = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0)
accumulator = accumulator * moe_weight[:, None]
accumulator = accumulator.to(compute_type)
# Write output
offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :]
c_mask = token_mask[:, None] & (offs_cn[None, :] < N)
tl.store(c_ptrs, accumulator, mask=c_mask)
def invoke_fused_moe_nvfp4_emulation_kernel(
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
B_scale: torch.Tensor,
act_global_scale: torch.Tensor,
w_global_scale: torch.Tensor,
topk_weights: torch.Tensor | None,
sorted_token_ids: torch.Tensor,
expert_ids: torch.Tensor,
num_tokens_post_padded: torch.Tensor,
mul_routed_weight: bool,
top_k: int,
config: dict[str, Any],
compute_type: tl.dtype,
):
"""Launch the fused NVFP4 emulation MoE kernel.
B has shape [E, N, K_packed] where K_packed = K // 2 (two FP4 per byte).
B_scale has shape [E, N, K // group_size] in FP8-E4M3 (stored as uint8).
w_global_scale has shape [E] (per-expert scalar).
"""
assert B_scale is not None and B_scale.ndim == 3
N = B.size(1)
K = A.size(1)
M = A.size(0)
num_tokens = M * top_k
EM = sorted_token_ids.size(0)
if A.size(0) < config["BLOCK_SIZE_M"]:
EM = min(
sorted_token_ids.size(0),
A.size(0) * top_k * config["BLOCK_SIZE_M"],
)
grid = lambda META: (
triton.cdiv(EM, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]),
)
fused_moe_nvfp4_emulation_kernel[grid](
A,
B,
C,
B_scale,
w_global_scale,
topk_weights,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
N,
K,
EM,
num_tokens,
A.stride(0),
A.stride(1),
# B is [E, N, K//2]: swap N and K strides so kernel indexes [K, N].
B.stride(0),
B.stride(2),
B.stride(1),
C.stride(1),
C.stride(2),
# B_scale is [E, N, K//group]: swap N and K strides likewise.
B_scale.stride(0),
B_scale.stride(2),
B_scale.stride(1),
block_k_diviable=K % config["BLOCK_SIZE_K"] == 0,
MUL_ROUTED_WEIGHT=mul_routed_weight,
top_k=top_k,
compute_type=compute_type,
group_size=16,
BLOCK_SIZE_M=config["BLOCK_SIZE_M"],
BLOCK_SIZE_N=config["BLOCK_SIZE_N"],
BLOCK_SIZE_K=config["BLOCK_SIZE_K"],
GROUP_SIZE_M=config["GROUP_SIZE_M"],
)
class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
"""
Extension of TritonExperts to support emulated NVFP4 MoE experts.
@@ -72,6 +372,31 @@ class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def supports_lora() -> bool:
return False
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
if moe_config.is_lora_enabled:
return False, "kernel does not support LoRA"
if moe_config.has_bias:
return False, "kernel does not support bias"
return TritonExperts.is_supported_config(
cls,
moe_config,
weight_key,
activation_key,
activation_format,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
@@ -109,47 +434,109 @@ class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
# w2 shape: [num_experts, hidden_size, intermediate_size//2]
assert w1.dtype == torch.uint8
assert w2.dtype == torch.uint8
assert hidden_states.is_contiguous()
assert hidden_states.dim() == 2
# Dequantize w1 from packed NVFP4 to fp16/bf16
w13_global_scale = self.quant_config.g1_alphas
K = hidden_states.size(-1)
assert w1.size(2) * 2 == K, f"Hidden size mismatch: {K} != {w1.size(2) * 2}"
w1_dequant = dequantize_to_dtype(
tensor_fp4=w1,
tensor_sf=self.w1_scale_val,
global_scale=w13_global_scale,
dtype=hidden_states.dtype,
block_size=16,
swizzle=False,
E, num_tokens, N, _, top_k_num = self.moe_problem_size(
hidden_states, w1, w2, topk_ids
)
# Dequantize w2 from packed NVFP4 to fp16/bf16
w2_global_scale = self.quant_config.g2_alphas
if global_num_experts == -1:
global_num_experts = E
w2_dequant = dequantize_to_dtype(
tensor_fp4=w2,
tensor_sf=self.w2_scale_val,
global_scale=w2_global_scale,
dtype=hidden_states.dtype,
block_size=16,
swizzle=False,
# TODO: There is actually no support for tuning of the underlying triton
# hyperparameters in benchmarks/kernels/benchmark_moe.py, to be added.
config = try_get_optimal_moe_config(
w1.size(),
w2.size(),
top_k_num,
self.quant_config.config_name(hidden_states.dtype),
num_tokens,
block_shape=None,
)
# Activation quantization/dequantization is deferred to
# `moe_kernel_quantize_input` in TritonExperts.apply.
super().apply(
output=output,
hidden_states=hidden_states,
w1=w1_dequant,
w2=w2_dequant,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
a1q_scale=None,
a2_scale=self.quant_config.a2_gscale,
workspace13=workspace13,
workspace2=workspace2,
expert_tokens_meta=expert_tokens_meta,
apply_router_weight_on_input=apply_router_weight_on_input,
if hidden_states.dtype == torch.bfloat16:
compute_type = tl.bfloat16
elif hidden_states.dtype == torch.float16:
compute_type = tl.float16
elif hidden_states.dtype == torch.float32:
compute_type = tl.float32
else:
raise ValueError(f"Unsupported compute_type: {hidden_states.dtype}")
intermediate_cache1 = _resize_cache(workspace2, (num_tokens, top_k_num, N))
activation_out_dim = self.adjust_N_for_activation(N, activation)
intermediate_cache2 = _resize_cache(
workspace13, (num_tokens * top_k_num, activation_out_dim)
)
intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K))
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
topk_ids,
config["BLOCK_SIZE_M"],
global_num_experts,
expert_map,
)
# Activation NVFP4 QDQ.
hidden_states_qdq, _ = moe_kernel_quantize_input(
A=hidden_states,
A_scale=self.quant_config.a1_gscale,
quant_dtype="nvfp4",
per_act_token_quant=False,
quantization_emulation=True,
)
# w13: fused weight dequant + GEMM.
invoke_fused_moe_nvfp4_emulation_kernel(
hidden_states_qdq,
w1,
intermediate_cache1,
self.w1_scale_val,
self.quant_config.a1_gscale,
self.quant_config.g1_alphas,
None, # topk_weights — applied after w2
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
False, # mul_routed_weight
top_k_num,
config,
compute_type=compute_type,
)
self.activation(
activation, intermediate_cache2, intermediate_cache1.view(-1, N)
)
# Activation NVFP4 QDQ.
intermediate_cache2_qdq, _ = moe_kernel_quantize_input(
A=intermediate_cache2,
A_scale=self.quant_config.a2_gscale,
quant_dtype="nvfp4",
per_act_token_quant=False,
quantization_emulation=True,
)
# w2: fused weight dequant + GEMM.
invoke_fused_moe_nvfp4_emulation_kernel(
intermediate_cache2_qdq,
w2,
intermediate_cache3,
self.w2_scale_val,
self.quant_config.a2_gscale,
self.quant_config.g2_alphas,
topk_weights,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
not apply_router_weight_on_input,
1,
config,
compute_type=compute_type,
)
self.moe_sum(intermediate_cache3, output)
@@ -23,28 +23,24 @@ kE2M1ToFloat_handle = SimpleNamespace(
@triton.jit
def _e2m1_inline(magnitude):
"""Inline E2M1 lookup using binary tree - 3 levels instead of 7 sequential.
def _e2m1_inline(nibble):
"""Decode an NVFP4 nibble (4 bits: 1 sign + 3 magnitude) to float32.
Maps 3-bit magnitude to float: [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
Uses bit decomposition for fewer comparisons.
Uses direct IEEE 754 bit construction.
For magnitudes 2-7 the FP32 bit pattern is 0x3F000000 + (mag << 22),
which is a single shift + add + bitcast. Magnitudes 0 (zero) and 1
(E2M1 subnormal = 0.5) are patched with two tl.where ops.
"""
# Bit 2 (MSB): separates 0-3 from 4-7
# Bit 1: separates within groups
# Bit 0 (LSB): separates within pairs
b2 = (magnitude >> 2) & 1 # 0 for mag 0-3, 1 for mag 4-7
b1 = (magnitude >> 1) & 1 # middle bit
b0 = magnitude & 1 # LSB
magnitude = nibble & 0x07
sign = (nibble >> 3) & 1
# For mag 0-3: [0.0, 0.5, 1.0, 1.5]
low_group = tl.where(
b1 == 1, tl.where(b0 == 1, 1.5, 1.0), tl.where(b0 == 1, 0.5, 0.0)
)
# For mag 4-7: [2.0, 3.0, 4.0, 6.0]
high_group = tl.where(
b1 == 1, tl.where(b0 == 1, 6.0, 4.0), tl.where(b0 == 1, 3.0, 2.0)
)
return tl.where(b2 == 1, high_group, low_group)
fp32_bits = 0x3F000000 + (magnitude.to(tl.int32) << 22)
val = fp32_bits.to(tl.float32, bitcast=True)
val = tl.where(magnitude == 0, 0.0, val)
val = tl.where(magnitude == 1, 0.5, val)
return tl.where(sign == 1, -val, val)
@triton.jit
@@ -65,7 +61,7 @@ def _dequantize_nvfp4_kernel(
"""
BLOCK_PACKED: tl.constexpr = BLOCK_SIZE // 2
row_idx = tl.program_id(0)
row_idx = tl.program_id(0).to(tl.int64)
tile_idx = tl.program_id(1)
if has_batch_global_scale:
@@ -105,16 +101,8 @@ def _dequantize_nvfp4_kernel(
low_nibble = raw_bytes & 0x0F
high_nibble = (raw_bytes >> 4) & 0x0F
# Binary tree E2M1 decode
low_mag = low_nibble & 0x07
low_val = _e2m1_inline(low_mag)
low_sign = (low_nibble >> 3) & 1
low_result = tl.where(low_sign == 1, -low_val, low_val) * scale_values
high_mag = high_nibble & 0x07
high_val = _e2m1_inline(high_mag)
high_sign = (high_nibble >> 3) & 1
high_result = tl.where(high_sign == 1, -high_val, high_val) * scale_values
low_result = _e2m1_inline(low_nibble) * scale_values
high_result = _e2m1_inline(high_nibble) * scale_values
# Interleave for coalesced contiguous store
result = tl.interleave(low_result, high_result)