From 9e137d8d230c9d62ba15c2a85f248ae7b0e20e2d Mon Sep 17 00:00:00 2001 From: Qidong Su Date: Tue, 8 Sep 2026 20:20:32 -0400 Subject: [PATCH] [Kernel] Enable optimized FlashInfer add-RMSNorm NVFP4 fusion (#51925) Signed-off-by: Qidong Su Signed-off-by: Qidong Su Co-authored-by: OpenAI Codex --- tests/compile/passes/test_fusion.py | 103 +++++++++++ .../passes/fusion/rms_quant_fusion.py | 169 ++++++++++++++++++ .../passes/utility/fix_functionalization.py | 14 ++ 3 files changed, 286 insertions(+) diff --git a/tests/compile/passes/test_fusion.py b/tests/compile/passes/test_fusion.py index 50c957fceb..843052f4af 100644 --- a/tests/compile/passes/test_fusion.py +++ b/tests/compile/passes/test_fusion.py @@ -9,14 +9,20 @@ import vllm.config import vllm.ir.ops import vllm.plugins from tests.compile.backend import TestBackend +from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor from tests.utils import TestFP8Layer from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops +from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS from vllm.compilation.passes.fusion.rms_quant_fusion import ( FUSED_OPS, FusedRMSQuantKey, RMSNormQuantFusionPass, ) +from vllm.compilation.passes.fx_utils import find_auto_fn_maybe +from vllm.compilation.passes.utility.fix_functionalization import ( + FixFunctionalizationPass, +) from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass from vllm.config import ( @@ -244,6 +250,31 @@ class TestModel(torch.nn.Module): ] +class AddRMSNormNvfp4Model(torch.nn.Module): + def __init__(self, hidden_size: int, eps: float): + super().__init__() + self.norm = RMSNorm(hidden_size, eps) + self.activation_scale = torch.rand((1, 1), dtype=torch.float32) + + weight = torch.rand((hidden_size, hidden_size)) + self.weight, self.weight_scale, weight_global_scale = quant_nvfp4_tensor(weight) + self.alpha = (1 / (weight_global_scale * self.activation_scale)).reshape(1) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + x = residual = torch.relu(x) + x, residual = self.norm(x, residual) + x, block_scale = scaled_fp4_quant(x, self.activation_scale) + x = cutlass_scaled_fp4_mm( + x, + self.weight, + block_scale, + self.weight_scale, + self.alpha, + out_dtype=torch.get_default_dtype(), + ) + return x, residual + + def _run_fusion_test( model, fusion_pass, @@ -384,6 +415,78 @@ def test_fusion_rmsnorm_quant( ) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("eps", [1e-5, 1e-6]) +@pytest.mark.parametrize( + ("num_tokens", "hidden_size"), + [(1, 256), (32, 1024), (257, 4096)], +) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") +def test_fusion_add_rmsnorm_nvfp4_quant( + dtype: torch.dtype, eps: float, num_tokens: int, hidden_size: int +): + fused_op = getattr( + torch.ops.vllm, + "flashinfer_fused_add_rms_norm_nvfp4_quant", + None, + ) + if not current_platform.has_device_capability(100) or fused_op is None: + pytest.skip("FlashInfer add-RMSNorm NVFP4 fusion is not available") + assert fused_op is not None + fused_op_default = fused_op.default + + vllm_config = VllmConfig( + model_config=ModelConfig(dtype=dtype), + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + pass_config=PassConfig( + fuse_norm_quant=True, + eliminate_noops=True, + ), + ), + ) + + with ( + vllm.config.set_current_vllm_config(vllm_config), + vllm_config.kernel_config.ir_op_priority.set_priority(), + ): + torch.set_default_device("cuda") + torch.set_default_dtype(dtype) + torch.manual_seed(1) + + fusion_pass = RMSNormQuantFusionPass(vllm_config) + fused_backend = TestBackend( + NoOpEliminationPass(vllm_config), + fusion_pass, + PostCleanupPass(vllm_config), + FixFunctionalizationPass(vllm_config), + ) + unfused_backend = TestBackend( + NoOpEliminationPass(vllm_config), + PostCleanupPass(vllm_config), + FixFunctionalizationPass(vllm_config), + ) + + model = AddRMSNormNvfp4Model(hidden_size=hidden_size, eps=eps) + x = torch.rand(num_tokens, hidden_size) + torch._dynamo.mark_dynamic(x, 0) + + result_fused = torch.compile(model, backend=fused_backend)(x) + result_unfused = torch.compile(model, backend=unfused_backend)(x) + + torch.testing.assert_close(result_fused, result_unfused, atol=2e-1, rtol=2e-1) + assert fusion_pass.matched_count == 1 + fused_backend.check_before_ops([torch.ops._C.scaled_fp4_quant.out]) + fused_backend.check_after_ops([fused_op_default]) + assert ( + find_auto_fn_maybe( + fused_backend.graph_post_pass.nodes, + fused_op_default, + ) + is None + ) + + @pytest.mark.parametrize("dtype", [torch.bfloat16]) @pytest.mark.parametrize("hidden_size", [256]) @pytest.mark.parametrize("num_tokens", [257]) diff --git a/vllm/compilation/passes/fusion/rms_quant_fusion.py b/vllm/compilation/passes/fusion/rms_quant_fusion.py index 6c55261e87..91c44f572c 100644 --- a/vllm/compilation/passes/fusion/rms_quant_fusion.py +++ b/vllm/compilation/passes/fusion/rms_quant_fusion.py @@ -26,6 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kStaticTensorScale, ) from vllm.platforms import current_platform +from vllm.utils.flashinfer import has_flashinfer +from vllm.utils.torch_utils import direct_register_custom_op from ..inductor_pass import enable_fake_mode from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass @@ -40,6 +42,73 @@ FP4_DTYPE = torch.uint8 _RMS_NORM_OP = torch.ops.vllm_ir.rms_norm.default _FUSED_ADD_RMS_NORM_OP = torch.ops.vllm_ir.fused_add_rms_norm.default +_FLASHINFER_ADD_RMSNORM_FP4QUANT: Any | None = None + + +def _flashinfer_fused_add_rms_norm_nvfp4_quant( + result: torch.Tensor, + result_block_scale: torch.Tensor, + residual: torch.Tensor, + input: torch.Tensor, + weight: torch.Tensor, + input_global_scale: torch.Tensor, + block_scale_unswizzled: torch.Tensor, + is_sf_swizzled_layout: bool, + epsilon: float, +) -> None: + """FlashInfer fused add + RMSNorm + NVFP4 quantization.""" + assert _FLASHINFER_ADD_RMSNORM_FP4QUANT is not None + _FLASHINFER_ADD_RMSNORM_FP4QUANT( + input, + residual, + weight, + y_fp4=result.view(torch.float4_e2m1fn_x2), + block_scale=result_block_scale.view(torch.float8_e4m3fn), + global_scale=input_global_scale.reshape(1), + eps=epsilon, + block_size=16, + scale_format="e4m3", + is_sf_swizzled_layout=is_sf_swizzled_layout, + output_both_sf_layouts=False, + block_scale_unswizzled=block_scale_unswizzled, + ) + + +def _flashinfer_fused_add_rms_norm_nvfp4_quant_fake( + result: torch.Tensor, + result_block_scale: torch.Tensor, + residual: torch.Tensor, + input: torch.Tensor, + weight: torch.Tensor, + input_global_scale: torch.Tensor, + block_scale_unswizzled: torch.Tensor, + is_sf_swizzled_layout: bool, + epsilon: float, +) -> None: + return None + + +_FLASHINFER_NVFP4_RMS_QUANT_OP: OpOverload | None = None +if ( + current_platform.is_cuda() + and hasattr(torch, "float4_e2m1fn_x2") + and has_flashinfer() +): + try: + from flashinfer import add_rmsnorm_fp4quant + except ImportError: + pass + else: + _FLASHINFER_ADD_RMSNORM_FP4QUANT = add_rmsnorm_fp4quant + direct_register_custom_op( + op_name="flashinfer_fused_add_rms_norm_nvfp4_quant", + op_func=_flashinfer_fused_add_rms_norm_nvfp4_quant, + mutates_args=["result", "result_block_scale", "residual"], + fake_impl=_flashinfer_fused_add_rms_norm_nvfp4_quant_fake, + ) + _FLASHINFER_NVFP4_RMS_QUANT_OP = ( + torch.ops.vllm.flashinfer_fused_add_rms_norm_nvfp4_quant.default + ) # TODO: extend rmsnorm quant kernels to support mixed input/weight dtypes, @@ -610,6 +679,97 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern): ) +class FusedAddRMSNormNvfp4QuantPattern: + """Fuse add-RMSNorm with NVFP4 quantization for either scale layout.""" + + def __init__(self, epsilon: float, is_sf_swizzled_layout: bool) -> None: + assert _FLASHINFER_NVFP4_RMS_QUANT_OP is not None + self.epsilon = epsilon + self.is_sf_swizzled_layout = is_sf_swizzled_layout + self.FUSED_OP = _FLASHINFER_NVFP4_RMS_QUANT_OP + + def register(self, pm_pass: PatternMatcherPass) -> None: + def pattern( + result: torch.Tensor, + result_block_scale: torch.Tensor, + input: torch.Tensor, + weight: torch.Tensor, + residual: torch.Tensor, + input_global_scale: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + result_rms, updated_residual = vllm.ir.ops.fused_add_rms_norm( + input, residual, weight, self.epsilon + ) + at = auto_functionalized( + torch.ops._C.scaled_fp4_quant.out, + input=result_rms, + input_scale=input_global_scale, + is_sf_swizzled_layout=self.is_sf_swizzled_layout, + output=result, + output_scale=result_block_scale, + ) + return at[1], updated_residual, at[2] + + def replacement( + result: torch.Tensor, + result_block_scale: torch.Tensor, + input: torch.Tensor, + weight: torch.Tensor, + residual: torch.Tensor, + input_global_scale: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + hidden_size = input.shape[-1] + num_tokens = input.numel() // hidden_size + # This full-size dummy is required by FlashInfer's TVM-FFI tensor + # validation even though output_both_sf_layouts=False leaves it untouched. + block_scale_unswizzled = torch.empty( + (num_tokens, hidden_size // 16), + dtype=torch.float8_e4m3fn, + device=input.device, + ) + at = auto_functionalized( + self.FUSED_OP, + result=result, + result_block_scale=result_block_scale, + residual=residual, + input=input, + weight=weight, + input_global_scale=input_global_scale, + block_scale_unswizzled=block_scale_unswizzled, + is_sf_swizzled_layout=self.is_sf_swizzled_layout, + epsilon=self.epsilon, + ) + # result, updated residual, block scale in the requested layout + return at[1], at[3], at[2] + + inputs = [ + torch.empty( + (5, 32), dtype=torch.uint8, device=current_platform.device_type + ), + ( + empty_i32(128, 4) + if self.is_sf_swizzled_layout + else torch.empty( + (5, 4), + dtype=torch.uint8, + device=current_platform.device_type, + ) + ), + empty_bf16(5, 64), + empty_bf16(64), + empty_bf16(5, 64), + empty_fp32(1), + ] + pm.register_replacement( + pattern, + replacement, + inputs, + pm.fwd_only, + pm_pass, + extra_check=_rms_input_weight_dtype_match, + ) + + class RMSNormQuantFusionPass(VllmPatternMatcherPass): """ This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op. @@ -627,6 +787,14 @@ class RMSNormQuantFusionPass(VllmPatternMatcherPass): # Make sure fused add patterns are before simple rms norm, # as the latter is a subset of the former in torch ops for epsilon in [1e-5, 1e-6]: + if _FLASHINFER_NVFP4_RMS_QUANT_OP is not None and ( + current_platform.has_device_capability(100) + ): + for is_sf_swizzled_layout in (True, False): + FusedAddRMSNormNvfp4QuantPattern( + epsilon, is_sf_swizzled_layout + ).register(self.patterns) + # Fuse fused_add_rms_norm + static fp8 quant FusedAddRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register( self.patterns @@ -685,4 +853,5 @@ class RMSNormQuantFusionPass(VllmPatternMatcherPass): FusedAddRMSNormStaticQuantPattern, FusedAddRMSNormDynamicQuantPattern, FusedAddRMSNormGroupQuantPattern, + FusedAddRMSNormNvfp4QuantPattern, ) diff --git a/vllm/compilation/passes/utility/fix_functionalization.py b/vllm/compilation/passes/utility/fix_functionalization.py index c0643a916b..c04f9abbb6 100644 --- a/vllm/compilation/passes/utility/fix_functionalization.py +++ b/vllm/compilation/passes/utility/fix_functionalization.py @@ -124,6 +124,20 @@ class FixFunctionalizationPass(VllmInductorPass): 5: "scale_out", } self.defunctionalize(graph, node, mutated_args) + elif ( + hasattr( + torch.ops.vllm, + "flashinfer_fused_add_rms_norm_nvfp4_quant", + ) + and at_target + == torch.ops.vllm.flashinfer_fused_add_rms_norm_nvfp4_quant.default + ): + mutated_args = { + 1: "result", + 2: "result_block_scale", + 3: "residual", + } + self.defunctionalize(graph, node, mutated_args) # For some reason we need to specify the args for both # silu_and_mul and silu_and_mul_quant. The kwargs # pathway gets the wrong answer.