[Quantization] Add per-token NVFP4 CuTe-DSL MoE backend (#50030)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: S1ro1 <matej.sirovatka@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
aoshen02
parent
d5ee309314
commit
0103a9b96f
@@ -14,11 +14,14 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
|
||||
activation_to_flashinfer_int,
|
||||
quantize_nvfp4_per_token_input,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4DynamicToken,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -40,6 +43,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
per_token_activation: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config=moe_config,
|
||||
@@ -63,6 +67,14 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
|
||||
self.situ_beta = moe_config.activation_situ_beta
|
||||
self.situ_linear_beta = moe_config.activation_situ_linear_beta
|
||||
self.per_token_activation = per_token_activation
|
||||
self.per_token_global_scale = None
|
||||
if per_token_activation:
|
||||
assert quant_config.a2_gscale is not None
|
||||
self.per_token_global_scale = quant_config.a2_gscale.new_full(
|
||||
(1,),
|
||||
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
|
||||
@@ -92,6 +104,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kNvfp4Static, kNvfp4Dynamic),
|
||||
(kNvfp4Static, kNvfp4DynamicToken),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
@@ -114,6 +127,10 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return self.per_token_activation
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
@@ -127,8 +144,8 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
workspace1 = (0,)
|
||||
workspace2 = (0,)
|
||||
# K is packed (K//2 for uint8), so output uses hidden_dim.
|
||||
assert self.hidden_dim == K * 2
|
||||
expected_hidden_dim = K if self.expects_unquantized_inputs else K * 2
|
||||
assert self.hidden_dim == expected_hidden_dim
|
||||
output = (M, self.hidden_dim)
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
@@ -151,13 +168,26 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
apply_router_weight_on_input: bool | None,
|
||||
):
|
||||
assert self.quant_dtype == "nvfp4"
|
||||
assert a1q_scale is not None
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
|
||||
if self.expects_unquantized_inputs:
|
||||
hidden_states, block_scale, per_token_scale = (
|
||||
quantize_nvfp4_per_token_input(hidden_states)
|
||||
)
|
||||
fc2_input_scale = self.per_token_global_scale
|
||||
else:
|
||||
assert a1q_scale is not None
|
||||
block_scale = a1q_scale
|
||||
per_token_scale = None
|
||||
fc2_input_scale = self.a2_gscale
|
||||
|
||||
assert block_scale is not None
|
||||
assert fc2_input_scale is not None
|
||||
|
||||
# a1q_scale is (M, K//16) float8_e4m3fn from fp4_quantize.
|
||||
# The functional API expects x_sf with trailing dim: (M, K//16, 1).
|
||||
x_sf = a1q_scale.unsqueeze(-1)
|
||||
x_sf = block_scale.unsqueeze(-1)
|
||||
|
||||
# The kernel defaults swiglu_{alpha,beta,limit} to the plain-SwiGLU
|
||||
# values, so only forward the ones the model actually sets.
|
||||
@@ -188,6 +218,9 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
}
|
||||
swiglu_kwargs = {k: v for k, v in swiglu_params.items() if v is not None}
|
||||
|
||||
per_token_kwargs = (
|
||||
{"per_token_scale": per_token_scale} if self.per_token_activation else {}
|
||||
)
|
||||
flashinfer_cute_dsl_fused_moe_nvfp4(
|
||||
x=hidden_states,
|
||||
x_sf=x_sf,
|
||||
@@ -196,7 +229,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
w1_weight=w1,
|
||||
w1_weight_sf=self.w1_scale,
|
||||
w1_alpha=self.g1_alphas,
|
||||
fc2_input_scale=self.a2_gscale,
|
||||
fc2_input_scale=fc2_input_scale,
|
||||
w2_weight=w2,
|
||||
w2_weight_sf=self.w2_scale,
|
||||
w2_alpha=self.g2_alphas,
|
||||
@@ -209,4 +242,5 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
MoEActivation.SILU if activation == MoEActivation.SITU else activation
|
||||
),
|
||||
**swiglu_kwargs,
|
||||
**per_token_kwargs,
|
||||
)
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
activation_to_flashinfer_int,
|
||||
has_flashinfer_situ_activation,
|
||||
quantize_nvfp4_per_token_input,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -37,10 +38,6 @@ from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Base scale for per-token NVFP4 activation quant; the kernel folds the
|
||||
# per-token global scale (from the activation amax) on top of it.
|
||||
_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
|
||||
|
||||
|
||||
class TrtLlmNvFp4ExpertsBase:
|
||||
"""NvFp4 TRTLLM-Gen MoE kernels. Supports modular and monolithic interface."""
|
||||
@@ -259,23 +256,6 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return self.per_token_activation
|
||||
|
||||
def _quantize_per_token_input(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""NVFP4-quantize activations with a per-token global scale.
|
||||
|
||||
Returns ``(packed_fp4, block_scale, per_token_scale)``.
|
||||
"""
|
||||
from flashinfer import SfLayout, nvfp4_quantize
|
||||
|
||||
hs_fp4, hs_block_scale, per_token_scale = nvfp4_quantize(
|
||||
hidden_states,
|
||||
_PER_TOKEN_BASE_GLOBAL_SCALE,
|
||||
sfLayout=SfLayout.layout_linear,
|
||||
per_token_activation=True,
|
||||
)
|
||||
return hs_fp4, hs_block_scale, per_token_scale
|
||||
|
||||
def _get_chunk_size(self) -> int:
|
||||
MAX_GRID_Y = 65535
|
||||
MAX_TILE_TOKENS_DIM = 128
|
||||
@@ -361,9 +341,9 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
|
||||
# Per-token: input is unquantized, quantize it here. Otherwise it was
|
||||
# already quantized in prepare() with the static global scale.
|
||||
if self.per_token_activation:
|
||||
if self.expects_unquantized_inputs:
|
||||
hidden_states, block_scale, per_token_scale = (
|
||||
self._quantize_per_token_input(hidden_states)
|
||||
quantize_nvfp4_per_token_input(hidden_states)
|
||||
)
|
||||
else:
|
||||
block_scale, per_token_scale = a1q_scale, None
|
||||
@@ -437,7 +417,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
) -> UnfinalizedMoEOutput | None:
|
||||
assert self._supports_activation(activation)
|
||||
# Per-token defers input quant to _invoke_kernel, so a1q_scale is None.
|
||||
assert a1q_scale is not None or self.per_token_activation
|
||||
assert a1q_scale is not None or self.expects_unquantized_inputs
|
||||
|
||||
# DeepEP produces int64 indexes.
|
||||
topk_ids = topk_ids.to(dtype=torch.int32)
|
||||
@@ -539,7 +519,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
import flashinfer
|
||||
|
||||
assert self._supports_activation(activation)
|
||||
assert a1q_scale is not None or self.per_token_activation
|
||||
assert a1q_scale is not None or self.expects_unquantized_inputs
|
||||
assert self.quant_config.w1_scale is not None
|
||||
assert self.quant_config.w2_scale is not None
|
||||
assert (
|
||||
@@ -551,9 +531,9 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
)
|
||||
|
||||
# Per-token: input is unquantized, quantize it here (see modular apply).
|
||||
if self.per_token_activation:
|
||||
if self.expects_unquantized_inputs:
|
||||
hidden_states, block_scale, per_token_scale = (
|
||||
self._quantize_per_token_input(hidden_states)
|
||||
quantize_nvfp4_per_token_input(hidden_states)
|
||||
)
|
||||
else:
|
||||
block_scale, per_token_scale = a1q_scale, None
|
||||
|
||||
Reference in New Issue
Block a user