[Kernel] Enable optimized FlashInfer add-RMSNorm NVFP4 fusion (#51925)

Signed-off-by: Qidong Su <qidongs@nvidia.com>
Signed-off-by: Qidong Su <soodoshll@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Qidong Su
2026-09-08 17:20:32 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent c268198715
commit 9e137d8d23
3 changed files with 286 additions and 0 deletions
+103
View File
@@ -9,14 +9,20 @@ import vllm.config
import vllm.ir.ops
import vllm.plugins
from tests.compile.backend import TestBackend
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
from tests.utils import TestFP8Layer
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
from vllm.compilation.passes.fusion.rms_quant_fusion import (
FUSED_OPS,
FusedRMSQuantKey,
RMSNormQuantFusionPass,
)
from vllm.compilation.passes.fx_utils import find_auto_fn_maybe
from vllm.compilation.passes.utility.fix_functionalization import (
FixFunctionalizationPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
@@ -244,6 +250,31 @@ class TestModel(torch.nn.Module):
]
class AddRMSNormNvfp4Model(torch.nn.Module):
def __init__(self, hidden_size: int, eps: float):
super().__init__()
self.norm = RMSNorm(hidden_size, eps)
self.activation_scale = torch.rand((1, 1), dtype=torch.float32)
weight = torch.rand((hidden_size, hidden_size))
self.weight, self.weight_scale, weight_global_scale = quant_nvfp4_tensor(weight)
self.alpha = (1 / (weight_global_scale * self.activation_scale)).reshape(1)
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
x = residual = torch.relu(x)
x, residual = self.norm(x, residual)
x, block_scale = scaled_fp4_quant(x, self.activation_scale)
x = cutlass_scaled_fp4_mm(
x,
self.weight,
block_scale,
self.weight_scale,
self.alpha,
out_dtype=torch.get_default_dtype(),
)
return x, residual
def _run_fusion_test(
model,
fusion_pass,
@@ -384,6 +415,78 @@ def test_fusion_rmsnorm_quant(
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize(
("num_tokens", "hidden_size"),
[(1, 256), (32, 1024), (257, 4096)],
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
def test_fusion_add_rmsnorm_nvfp4_quant(
dtype: torch.dtype, eps: float, num_tokens: int, hidden_size: int
):
fused_op = getattr(
torch.ops.vllm,
"flashinfer_fused_add_rms_norm_nvfp4_quant",
None,
)
if not current_platform.has_device_capability(100) or fused_op is None:
pytest.skip("FlashInfer add-RMSNorm NVFP4 fusion is not available")
assert fused_op is not None
fused_op_default = fused_op.default
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(
fuse_norm_quant=True,
eliminate_noops=True,
),
),
)
with (
vllm.config.set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
fusion_pass = RMSNormQuantFusionPass(vllm_config)
fused_backend = TestBackend(
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
FixFunctionalizationPass(vllm_config),
)
unfused_backend = TestBackend(
NoOpEliminationPass(vllm_config),
PostCleanupPass(vllm_config),
FixFunctionalizationPass(vllm_config),
)
model = AddRMSNormNvfp4Model(hidden_size=hidden_size, eps=eps)
x = torch.rand(num_tokens, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
result_fused = torch.compile(model, backend=fused_backend)(x)
result_unfused = torch.compile(model, backend=unfused_backend)(x)
torch.testing.assert_close(result_fused, result_unfused, atol=2e-1, rtol=2e-1)
assert fusion_pass.matched_count == 1
fused_backend.check_before_ops([torch.ops._C.scaled_fp4_quant.out])
fused_backend.check_after_ops([fused_op_default])
assert (
find_auto_fn_maybe(
fused_backend.graph_post_pass.nodes,
fused_op_default,
)
is None
)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [256])
@pytest.mark.parametrize("num_tokens", [257])
@@ -26,6 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kStaticTensorScale,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.torch_utils import direct_register_custom_op
from ..inductor_pass import enable_fake_mode
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
@@ -40,6 +42,73 @@ FP4_DTYPE = torch.uint8
_RMS_NORM_OP = torch.ops.vllm_ir.rms_norm.default
_FUSED_ADD_RMS_NORM_OP = torch.ops.vllm_ir.fused_add_rms_norm.default
_FLASHINFER_ADD_RMSNORM_FP4QUANT: Any | None = None
def _flashinfer_fused_add_rms_norm_nvfp4_quant(
result: torch.Tensor,
result_block_scale: torch.Tensor,
residual: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
input_global_scale: torch.Tensor,
block_scale_unswizzled: torch.Tensor,
is_sf_swizzled_layout: bool,
epsilon: float,
) -> None:
"""FlashInfer fused add + RMSNorm + NVFP4 quantization."""
assert _FLASHINFER_ADD_RMSNORM_FP4QUANT is not None
_FLASHINFER_ADD_RMSNORM_FP4QUANT(
input,
residual,
weight,
y_fp4=result.view(torch.float4_e2m1fn_x2),
block_scale=result_block_scale.view(torch.float8_e4m3fn),
global_scale=input_global_scale.reshape(1),
eps=epsilon,
block_size=16,
scale_format="e4m3",
is_sf_swizzled_layout=is_sf_swizzled_layout,
output_both_sf_layouts=False,
block_scale_unswizzled=block_scale_unswizzled,
)
def _flashinfer_fused_add_rms_norm_nvfp4_quant_fake(
result: torch.Tensor,
result_block_scale: torch.Tensor,
residual: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
input_global_scale: torch.Tensor,
block_scale_unswizzled: torch.Tensor,
is_sf_swizzled_layout: bool,
epsilon: float,
) -> None:
return None
_FLASHINFER_NVFP4_RMS_QUANT_OP: OpOverload | None = None
if (
current_platform.is_cuda()
and hasattr(torch, "float4_e2m1fn_x2")
and has_flashinfer()
):
try:
from flashinfer import add_rmsnorm_fp4quant
except ImportError:
pass
else:
_FLASHINFER_ADD_RMSNORM_FP4QUANT = add_rmsnorm_fp4quant
direct_register_custom_op(
op_name="flashinfer_fused_add_rms_norm_nvfp4_quant",
op_func=_flashinfer_fused_add_rms_norm_nvfp4_quant,
mutates_args=["result", "result_block_scale", "residual"],
fake_impl=_flashinfer_fused_add_rms_norm_nvfp4_quant_fake,
)
_FLASHINFER_NVFP4_RMS_QUANT_OP = (
torch.ops.vllm.flashinfer_fused_add_rms_norm_nvfp4_quant.default
)
# TODO: extend rmsnorm quant kernels to support mixed input/weight dtypes,
@@ -610,6 +679,97 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
)
class FusedAddRMSNormNvfp4QuantPattern:
"""Fuse add-RMSNorm with NVFP4 quantization for either scale layout."""
def __init__(self, epsilon: float, is_sf_swizzled_layout: bool) -> None:
assert _FLASHINFER_NVFP4_RMS_QUANT_OP is not None
self.epsilon = epsilon
self.is_sf_swizzled_layout = is_sf_swizzled_layout
self.FUSED_OP = _FLASHINFER_NVFP4_RMS_QUANT_OP
def register(self, pm_pass: PatternMatcherPass) -> None:
def pattern(
result: torch.Tensor,
result_block_scale: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
input_global_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms, updated_residual = vllm.ir.ops.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
at = auto_functionalized(
torch.ops._C.scaled_fp4_quant.out,
input=result_rms,
input_scale=input_global_scale,
is_sf_swizzled_layout=self.is_sf_swizzled_layout,
output=result,
output_scale=result_block_scale,
)
return at[1], updated_residual, at[2]
def replacement(
result: torch.Tensor,
result_block_scale: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
input_global_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hidden_size = input.shape[-1]
num_tokens = input.numel() // hidden_size
# This full-size dummy is required by FlashInfer's TVM-FFI tensor
# validation even though output_both_sf_layouts=False leaves it untouched.
block_scale_unswizzled = torch.empty(
(num_tokens, hidden_size // 16),
dtype=torch.float8_e4m3fn,
device=input.device,
)
at = auto_functionalized(
self.FUSED_OP,
result=result,
result_block_scale=result_block_scale,
residual=residual,
input=input,
weight=weight,
input_global_scale=input_global_scale,
block_scale_unswizzled=block_scale_unswizzled,
is_sf_swizzled_layout=self.is_sf_swizzled_layout,
epsilon=self.epsilon,
)
# result, updated residual, block scale in the requested layout
return at[1], at[3], at[2]
inputs = [
torch.empty(
(5, 32), dtype=torch.uint8, device=current_platform.device_type
),
(
empty_i32(128, 4)
if self.is_sf_swizzled_layout
else torch.empty(
(5, 4),
dtype=torch.uint8,
device=current_platform.device_type,
)
),
empty_bf16(5, 64),
empty_bf16(64),
empty_bf16(5, 64),
empty_fp32(1),
]
pm.register_replacement(
pattern,
replacement,
inputs,
pm.fwd_only,
pm_pass,
extra_check=_rms_input_weight_dtype_match,
)
class RMSNormQuantFusionPass(VllmPatternMatcherPass):
"""
This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
@@ -627,6 +787,14 @@ class RMSNormQuantFusionPass(VllmPatternMatcherPass):
# Make sure fused add patterns are before simple rms norm,
# as the latter is a subset of the former in torch ops
for epsilon in [1e-5, 1e-6]:
if _FLASHINFER_NVFP4_RMS_QUANT_OP is not None and (
current_platform.has_device_capability(100)
):
for is_sf_swizzled_layout in (True, False):
FusedAddRMSNormNvfp4QuantPattern(
epsilon, is_sf_swizzled_layout
).register(self.patterns)
# Fuse fused_add_rms_norm + static fp8 quant
FusedAddRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(
self.patterns
@@ -685,4 +853,5 @@ class RMSNormQuantFusionPass(VllmPatternMatcherPass):
FusedAddRMSNormStaticQuantPattern,
FusedAddRMSNormDynamicQuantPattern,
FusedAddRMSNormGroupQuantPattern,
FusedAddRMSNormNvfp4QuantPattern,
)
@@ -124,6 +124,20 @@ class FixFunctionalizationPass(VllmInductorPass):
5: "scale_out",
}
self.defunctionalize(graph, node, mutated_args)
elif (
hasattr(
torch.ops.vllm,
"flashinfer_fused_add_rms_norm_nvfp4_quant",
)
and at_target
== torch.ops.vllm.flashinfer_fused_add_rms_norm_nvfp4_quant.default
):
mutated_args = {
1: "result",
2: "result_block_scale",
3: "residual",
}
self.defunctionalize(graph, node, mutated_args)
# For some reason we need to specify the args for both
# silu_and_mul and silu_and_mul_quant. The kwargs
# pathway gets the wrong answer.