[Quant] Add nvfp4_per_token online MoE quantization (#48538)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-07-16 14:25:27 -07:00
committed by GitHub
parent ab3c1aedf3
commit c95c663049
7 changed files with 285 additions and 5 deletions
@@ -33,6 +33,10 @@ from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
logger = init_logger(__name__)
# Base scale for per-token NVFP4 activation quant; the kernel folds the
# per-token global scale (from the activation amax) on top of it.
_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
class TrtLlmNvFp4ExpertsBase:
"""
@@ -43,9 +47,13 @@ class TrtLlmNvFp4ExpertsBase:
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
per_token_activation: bool = False,
):
self.moe_config = moe_config
self.quant_config = quant_config
# Quantize the input here (deferred from prepare) to capture a per-token
# global scale, instead of a static one.
self.per_token_activation = per_token_activation
self.routing_method_type = self.moe_config.routing_method
self.topk = moe_config.experts_per_token
@@ -211,6 +219,27 @@ class TrtLlmNvFp4ExpertsBase:
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@property
def expects_unquantized_inputs(self) -> bool:
return self.per_token_activation
def _quantize_per_token_input(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""NVFP4-quantize activations with a per-token global scale.
Returns ``(packed_fp4, block_scale, per_token_scale)``.
"""
from flashinfer import SfLayout, nvfp4_quantize
hs_fp4, hs_block_scale, per_token_scale = nvfp4_quantize(
hidden_states,
_PER_TOKEN_BASE_GLOBAL_SCALE,
sfLayout=SfLayout.layout_linear,
per_token_activation=True,
)
return hs_fp4, hs_block_scale, per_token_scale
def _get_chunk_size(self) -> int:
MAX_GRID_Y = 65535
MAX_TILE_TOKENS_DIM = 128
@@ -255,6 +284,15 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
if self.per_token_activation:
# Deferred input quant leaves K unpacked here, breaking the
# workspace assumptions below. Per-token NVFP4 is only supported on
# the monolithic (non-EP) path for now.
raise NotImplementedError(
"NVFP4 per-token activation is only supported on the monolithic "
"(non-EP) FlashInfer TRTLLM MoE path."
)
# The workspaces for this implementation are managed by flashinfer.
workspace1 = (0,)
workspace2 = (0,)
@@ -286,6 +324,15 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
assert self.quant_config.w1_scale is not None
assert self.quant_config.w2_scale is not None
# Per-token: input is unquantized, quantize it here. Otherwise it was
# already quantized in prepare() with the static global scale.
if self.per_token_activation:
hidden_states, block_scale, per_token_scale = (
self._quantize_per_token_input(hidden_states)
)
else:
block_scale, per_token_scale = a1q_scale, None
# Pack topk ids and weights into format expected by the kernel.
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
output1_scale_gate_scalar = self.quant_config.g1_alphas
@@ -295,7 +342,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
topk_ids=packed_tensor,
routing_bias=None,
hidden_states=hidden_states,
hidden_states_scale=a1q_scale.view(torch.float8_e4m3fn).reshape(
hidden_states_scale=block_scale.view(torch.float8_e4m3fn).reshape(
*hidden_states.shape[:-1], -1
),
gemm1_weights=w1,
@@ -321,6 +368,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
routing_method_type=1, # not used
do_finalize=True,
activation_type=activation_to_flashinfer_int(activation),
per_token_scale=per_token_scale,
output=output,
tune_max_num_tokens=min(
fi_moe_largest_bucket(self.moe_config), self._get_chunk_size()
@@ -346,7 +394,8 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
apply_router_weight_on_input: bool,
):
assert self._supports_activation(activation)
assert a1q_scale is not None
# Per-token defers input quant to _invoke_kernel, so a1q_scale is None.
assert a1q_scale is not None or self.per_token_activation
M = hidden_states.shape[0]
chunk_size = self._get_chunk_size()
@@ -375,7 +424,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
topk_ids[start:end],
activation,
global_num_experts,
a1q_scale[start:end],
None if a1q_scale is None else a1q_scale[start:end],
)
@@ -439,7 +488,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
import flashinfer
assert self._supports_activation(activation)
assert a1q_scale is not None
assert a1q_scale is not None or self.per_token_activation
assert self.quant_config.w1_scale is not None
assert self.quant_config.w2_scale is not None
assert (
@@ -450,6 +499,14 @@ class TrtLlmNvFp4ExpertsMonolithic(
and self.routing_method_type != RoutingMethodType.Llama4
)
# Per-token: input is unquantized, quantize it here (see modular apply).
if self.per_token_activation:
hidden_states, block_scale, per_token_scale = (
self._quantize_per_token_input(hidden_states)
)
else:
block_scale, per_token_scale = a1q_scale, None
output1_scale_gate_scalar = self.quant_config.g1_alphas
# Invoke kernel.
@@ -459,7 +516,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
hidden_states=hidden_states,
hidden_states_scale=a1q_scale.view(torch.float8_e4m3fn).reshape(
hidden_states_scale=block_scale.view(torch.float8_e4m3fn).reshape(
*hidden_states.shape[:-1], -1
),
gemm1_weights=w1,
@@ -485,5 +542,6 @@ class TrtLlmNvFp4ExpertsMonolithic(
routing_method_type=self.routing_method_type,
do_finalize=True,
activation_type=activation_to_flashinfer_int(activation),
per_token_scale=per_token_scale,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)[0]
@@ -529,6 +529,7 @@ def make_nvfp4_moe_kernel(
backend: NvFp4MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: torch.nn.Module | None = None,
per_token_activation: bool = False,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -546,6 +547,8 @@ def make_nvfp4_moe_kernel(
if backend == NvFp4MoeBackend.HUMMING:
assert layer is not None
extra_kwargs = {"layer": layer}
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM and per_token_activation:
extra_kwargs["per_token_activation"] = True
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: