[Kernel] Enable optimized FlashInfer add-RMSNorm NVFP4 fusion (#51925)

Signed-off-by: Qidong Su <qidongs@nvidia.com>
Signed-off-by: Qidong Su <soodoshll@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Qidong Su
2026-09-08 17:20:32 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent c268198715
commit 9e137d8d23
3 changed files with 286 additions and 0 deletions
@@ -26,6 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kStaticTensorScale,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.torch_utils import direct_register_custom_op
from ..inductor_pass import enable_fake_mode
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
@@ -40,6 +42,73 @@ FP4_DTYPE = torch.uint8
_RMS_NORM_OP = torch.ops.vllm_ir.rms_norm.default
_FUSED_ADD_RMS_NORM_OP = torch.ops.vllm_ir.fused_add_rms_norm.default
_FLASHINFER_ADD_RMSNORM_FP4QUANT: Any | None = None
def _flashinfer_fused_add_rms_norm_nvfp4_quant(
result: torch.Tensor,
result_block_scale: torch.Tensor,
residual: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
input_global_scale: torch.Tensor,
block_scale_unswizzled: torch.Tensor,
is_sf_swizzled_layout: bool,
epsilon: float,
) -> None:
"""FlashInfer fused add + RMSNorm + NVFP4 quantization."""
assert _FLASHINFER_ADD_RMSNORM_FP4QUANT is not None
_FLASHINFER_ADD_RMSNORM_FP4QUANT(
input,
residual,
weight,
y_fp4=result.view(torch.float4_e2m1fn_x2),
block_scale=result_block_scale.view(torch.float8_e4m3fn),
global_scale=input_global_scale.reshape(1),
eps=epsilon,
block_size=16,
scale_format="e4m3",
is_sf_swizzled_layout=is_sf_swizzled_layout,
output_both_sf_layouts=False,
block_scale_unswizzled=block_scale_unswizzled,
)
def _flashinfer_fused_add_rms_norm_nvfp4_quant_fake(
result: torch.Tensor,
result_block_scale: torch.Tensor,
residual: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
input_global_scale: torch.Tensor,
block_scale_unswizzled: torch.Tensor,
is_sf_swizzled_layout: bool,
epsilon: float,
) -> None:
return None
_FLASHINFER_NVFP4_RMS_QUANT_OP: OpOverload | None = None
if (
current_platform.is_cuda()
and hasattr(torch, "float4_e2m1fn_x2")
and has_flashinfer()
):
try:
from flashinfer import add_rmsnorm_fp4quant
except ImportError:
pass
else:
_FLASHINFER_ADD_RMSNORM_FP4QUANT = add_rmsnorm_fp4quant
direct_register_custom_op(
op_name="flashinfer_fused_add_rms_norm_nvfp4_quant",
op_func=_flashinfer_fused_add_rms_norm_nvfp4_quant,
mutates_args=["result", "result_block_scale", "residual"],
fake_impl=_flashinfer_fused_add_rms_norm_nvfp4_quant_fake,
)
_FLASHINFER_NVFP4_RMS_QUANT_OP = (
torch.ops.vllm.flashinfer_fused_add_rms_norm_nvfp4_quant.default
)
# TODO: extend rmsnorm quant kernels to support mixed input/weight dtypes,
@@ -610,6 +679,97 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
)
class FusedAddRMSNormNvfp4QuantPattern:
"""Fuse add-RMSNorm with NVFP4 quantization for either scale layout."""
def __init__(self, epsilon: float, is_sf_swizzled_layout: bool) -> None:
assert _FLASHINFER_NVFP4_RMS_QUANT_OP is not None
self.epsilon = epsilon
self.is_sf_swizzled_layout = is_sf_swizzled_layout
self.FUSED_OP = _FLASHINFER_NVFP4_RMS_QUANT_OP
def register(self, pm_pass: PatternMatcherPass) -> None:
def pattern(
result: torch.Tensor,
result_block_scale: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
input_global_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms, updated_residual = vllm.ir.ops.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
at = auto_functionalized(
torch.ops._C.scaled_fp4_quant.out,
input=result_rms,
input_scale=input_global_scale,
is_sf_swizzled_layout=self.is_sf_swizzled_layout,
output=result,
output_scale=result_block_scale,
)
return at[1], updated_residual, at[2]
def replacement(
result: torch.Tensor,
result_block_scale: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
input_global_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hidden_size = input.shape[-1]
num_tokens = input.numel() // hidden_size
# This full-size dummy is required by FlashInfer's TVM-FFI tensor
# validation even though output_both_sf_layouts=False leaves it untouched.
block_scale_unswizzled = torch.empty(
(num_tokens, hidden_size // 16),
dtype=torch.float8_e4m3fn,
device=input.device,
)
at = auto_functionalized(
self.FUSED_OP,
result=result,
result_block_scale=result_block_scale,
residual=residual,
input=input,
weight=weight,
input_global_scale=input_global_scale,
block_scale_unswizzled=block_scale_unswizzled,
is_sf_swizzled_layout=self.is_sf_swizzled_layout,
epsilon=self.epsilon,
)
# result, updated residual, block scale in the requested layout
return at[1], at[3], at[2]
inputs = [
torch.empty(
(5, 32), dtype=torch.uint8, device=current_platform.device_type
),
(
empty_i32(128, 4)
if self.is_sf_swizzled_layout
else torch.empty(
(5, 4),
dtype=torch.uint8,
device=current_platform.device_type,
)
),
empty_bf16(5, 64),
empty_bf16(64),
empty_bf16(5, 64),
empty_fp32(1),
]
pm.register_replacement(
pattern,
replacement,
inputs,
pm.fwd_only,
pm_pass,
extra_check=_rms_input_weight_dtype_match,
)
class RMSNormQuantFusionPass(VllmPatternMatcherPass):
"""
This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
@@ -627,6 +787,14 @@ class RMSNormQuantFusionPass(VllmPatternMatcherPass):
# Make sure fused add patterns are before simple rms norm,
# as the latter is a subset of the former in torch ops
for epsilon in [1e-5, 1e-6]:
if _FLASHINFER_NVFP4_RMS_QUANT_OP is not None and (
current_platform.has_device_capability(100)
):
for is_sf_swizzled_layout in (True, False):
FusedAddRMSNormNvfp4QuantPattern(
epsilon, is_sf_swizzled_layout
).register(self.patterns)
# Fuse fused_add_rms_norm + static fp8 quant
FusedAddRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(
self.patterns
@@ -685,4 +853,5 @@ class RMSNormQuantFusionPass(VllmPatternMatcherPass):
FusedAddRMSNormStaticQuantPattern,
FusedAddRMSNormDynamicQuantPattern,
FusedAddRMSNormGroupQuantPattern,
FusedAddRMSNormNvfp4QuantPattern,
)
@@ -124,6 +124,20 @@ class FixFunctionalizationPass(VllmInductorPass):
5: "scale_out",
}
self.defunctionalize(graph, node, mutated_args)
elif (
hasattr(
torch.ops.vllm,
"flashinfer_fused_add_rms_norm_nvfp4_quant",
)
and at_target
== torch.ops.vllm.flashinfer_fused_add_rms_norm_nvfp4_quant.default
):
mutated_args = {
1: "result",
2: "result_block_scale",
3: "residual",
}
self.defunctionalize(graph, node, mutated_args)
# For some reason we need to specify the args for both
# silu_and_mul and silu_and_mul_quant. The kwargs
# pathway gets the wrong answer.