Enable B12x backend for non-gated MoEs (like Nemotron) (#43328)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-07-06 12:40:07 -07:00
committed by GitHub
co-authored by Andrii Skliar
parent b136cc2c2c
commit b1384f5ec6
3 changed files with 225 additions and 40 deletions
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -20,7 +22,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
flashinfer_b12x_fused_moe,
flashinfer_convert_sf_to_mma_layout,
has_flashinfer_b12x_moe,
)
@@ -42,6 +43,11 @@ class FlashInferB12xExperts(mk.FusedMoEExpertsModular):
Only NVFP4 (kNvfp4Static/kNvfp4Dynamic) quantization is supported.
"""
_ACTIVATION_MAP: dict[MoEActivation, str] = {
MoEActivation.SILU: "silu",
MoEActivation.RELU2_NO_MUL: "relu2",
}
def __init__(
self,
moe_config: FusedMoEConfig,
@@ -60,6 +66,30 @@ class FlashInferB12xExperts(mk.FusedMoEExpertsModular):
# one. Holding it on the instance keeps apply() alloc-free.
self._fc2_input_scale: torch.Tensor | None = None
# Shape params for B12xMoEWrapper construction.
self.global_num_experts = moe_config.num_experts
self.topk = moe_config.experts_per_token
self.hidden_dim = moe_config.hidden_dim
self.intermediate_size_per_partition = (
moe_config.intermediate_size_per_partition
)
self.max_num_tokens = moe_config.max_num_tokens
self.local_expert_offset = self.ep_rank * self.num_local_experts
activation = moe_config.activation
if activation not in self._ACTIVATION_MAP:
raise ValueError(
f"FlashInferB12xExperts does not support "
f"activation {activation!r}. "
f"Supported: {list(self._ACTIVATION_MAP.keys())}"
)
self._activation_str = self._ACTIVATION_MAP[activation]
# Lazily created on first apply() call.
self._wrapper: Any | None = None
self.w1_sf_mma: torch.Tensor | None = None
self.w2_sf_mma: torch.Tensor | None = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Normalise block scales to absorb the per-expert weight global scale
# (w_gs). vLLM's NVFP4 convention stores:
@@ -141,7 +171,7 @@ class FlashInferB12xExperts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
return True
@staticmethod
def _supports_quant_scheme(
@@ -158,11 +188,13 @@ class FlashInferB12xExperts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
return activation in (MoEActivation.SILU, MoEActivation.RELU2_NO_MUL)
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return True
# B12xMoEWrapper does not yet support expert parallelism: its local
# expert count must equal the global expert count.
return not moe_parallel_config.use_ep
def supports_expert_map(self) -> bool:
return False
@@ -190,13 +222,29 @@ class FlashInferB12xExperts(mk.FusedMoEExpertsModular):
@property
def expects_unquantized_inputs(self) -> bool:
# b12x_fused_moe expects BF16 hidden states and performs its own FP4
# B12xMoEWrapper expects BF16 hidden states and performs its own FP4
# quantization internally. Returning True prevents the modular kernel
# from pre-quantizing activations, which would produce an FP4-packed
# tensor with size(-1)=k//2 and break the scale-factor conversion that
# expects size(-1)=k.
# from pre-quantizing activations.
return True
def _ensure_wrapper(self) -> None:
"""Lazily create B12xMoEWrapper on first use."""
if self._wrapper is not None:
return
from flashinfer.fused_moe import B12xMoEWrapper
self._wrapper = B12xMoEWrapper(
num_experts=self.global_num_experts,
top_k=self.topk,
hidden_size=self.hidden_dim,
intermediate_size=self.intermediate_size_per_partition,
use_cuda_graph=True,
max_num_tokens=self.max_num_tokens,
num_local_experts=self.num_local_experts,
activation=self._activation_str,
)
def apply(
self,
output: torch.Tensor,
@@ -224,13 +272,16 @@ class FlashInferB12xExperts(mk.FusedMoEExpertsModular):
assert self._fc2_input_scale is not None, (
"_fc2_input_scale must be set by process_weights_after_loading"
)
assert self.w1_sf_mma is not None and self.w2_sf_mma is not None, (
"process_weights_after_loading must run before FlashInferB12xExperts.apply"
)
top_k = topk_ids.shape[1]
self._ensure_wrapper()
wrapper = self._wrapper
assert wrapper is not None
flashinfer_b12x_fused_moe(
wrapper_output = wrapper.run(
x=hidden_states,
token_selected_experts=topk_ids.to(torch.int32),
token_final_scales=topk_weights,
w1_weight=w1,
w1_weight_sf=self.w1_sf_mma,
w1_alpha=self.g1_alphas,
@@ -238,9 +289,7 @@ class FlashInferB12xExperts(mk.FusedMoEExpertsModular):
w2_weight=w2,
w2_weight_sf=self.w2_sf_mma,
w2_alpha=self.g2_alphas,
num_experts=global_num_experts,
top_k=top_k,
num_local_experts=self.num_local_experts,
output_dtype=self.out_dtype,
output=output,
token_selected_experts=topk_ids.to(torch.int32),
token_final_scales=topk_weights,
)
output.copy_(wrapper_output)