[Kernel] Enable optimized FlashInfer add-RMSNorm NVFP4 fusion (#51925)
Signed-off-by: Qidong Su <qidongs@nvidia.com> Signed-off-by: Qidong Su <soodoshll@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
c268198715
commit
9e137d8d23
@@ -9,14 +9,20 @@ import vllm.config
|
||||
import vllm.ir.ops
|
||||
import vllm.plugins
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
|
||||
from tests.utils import TestFP8Layer
|
||||
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
|
||||
from vllm.compilation.passes.fusion.rms_quant_fusion import (
|
||||
FUSED_OPS,
|
||||
FusedRMSQuantKey,
|
||||
RMSNormQuantFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fx_utils import find_auto_fn_maybe
|
||||
from vllm.compilation.passes.utility.fix_functionalization import (
|
||||
FixFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
@@ -244,6 +250,31 @@ class TestModel(torch.nn.Module):
|
||||
]
|
||||
|
||||
|
||||
class AddRMSNormNvfp4Model(torch.nn.Module):
|
||||
def __init__(self, hidden_size: int, eps: float):
|
||||
super().__init__()
|
||||
self.norm = RMSNorm(hidden_size, eps)
|
||||
self.activation_scale = torch.rand((1, 1), dtype=torch.float32)
|
||||
|
||||
weight = torch.rand((hidden_size, hidden_size))
|
||||
self.weight, self.weight_scale, weight_global_scale = quant_nvfp4_tensor(weight)
|
||||
self.alpha = (1 / (weight_global_scale * self.activation_scale)).reshape(1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x = residual = torch.relu(x)
|
||||
x, residual = self.norm(x, residual)
|
||||
x, block_scale = scaled_fp4_quant(x, self.activation_scale)
|
||||
x = cutlass_scaled_fp4_mm(
|
||||
x,
|
||||
self.weight,
|
||||
block_scale,
|
||||
self.weight_scale,
|
||||
self.alpha,
|
||||
out_dtype=torch.get_default_dtype(),
|
||||
)
|
||||
return x, residual
|
||||
|
||||
|
||||
def _run_fusion_test(
|
||||
model,
|
||||
fusion_pass,
|
||||
@@ -384,6 +415,78 @@ def test_fusion_rmsnorm_quant(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "hidden_size"),
|
||||
[(1, 256), (32, 1024), (257, 4096)],
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
def test_fusion_add_rmsnorm_nvfp4_quant(
|
||||
dtype: torch.dtype, eps: float, num_tokens: int, hidden_size: int
|
||||
):
|
||||
fused_op = getattr(
|
||||
torch.ops.vllm,
|
||||
"flashinfer_fused_add_rms_norm_nvfp4_quant",
|
||||
None,
|
||||
)
|
||||
if not current_platform.has_device_capability(100) or fused_op is None:
|
||||
pytest.skip("FlashInfer add-RMSNorm NVFP4 fusion is not available")
|
||||
assert fused_op is not None
|
||||
fused_op_default = fused_op.default
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with (
|
||||
vllm.config.set_current_vllm_config(vllm_config),
|
||||
vllm_config.kernel_config.ir_op_priority.set_priority(),
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(1)
|
||||
|
||||
fusion_pass = RMSNormQuantFusionPass(vllm_config)
|
||||
fused_backend = TestBackend(
|
||||
NoOpEliminationPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
FixFunctionalizationPass(vllm_config),
|
||||
)
|
||||
unfused_backend = TestBackend(
|
||||
NoOpEliminationPass(vllm_config),
|
||||
PostCleanupPass(vllm_config),
|
||||
FixFunctionalizationPass(vllm_config),
|
||||
)
|
||||
|
||||
model = AddRMSNormNvfp4Model(hidden_size=hidden_size, eps=eps)
|
||||
x = torch.rand(num_tokens, hidden_size)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
result_fused = torch.compile(model, backend=fused_backend)(x)
|
||||
result_unfused = torch.compile(model, backend=unfused_backend)(x)
|
||||
|
||||
torch.testing.assert_close(result_fused, result_unfused, atol=2e-1, rtol=2e-1)
|
||||
assert fusion_pass.matched_count == 1
|
||||
fused_backend.check_before_ops([torch.ops._C.scaled_fp4_quant.out])
|
||||
fused_backend.check_after_ops([fused_op_default])
|
||||
assert (
|
||||
find_auto_fn_maybe(
|
||||
fused_backend.graph_post_pass.nodes,
|
||||
fused_op_default,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("hidden_size", [256])
|
||||
@pytest.mark.parametrize("num_tokens", [257])
|
||||
|
||||
@@ -26,6 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kStaticTensorScale,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from ..inductor_pass import enable_fake_mode
|
||||
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
@@ -40,6 +42,73 @@ FP4_DTYPE = torch.uint8
|
||||
|
||||
_RMS_NORM_OP = torch.ops.vllm_ir.rms_norm.default
|
||||
_FUSED_ADD_RMS_NORM_OP = torch.ops.vllm_ir.fused_add_rms_norm.default
|
||||
_FLASHINFER_ADD_RMSNORM_FP4QUANT: Any | None = None
|
||||
|
||||
|
||||
def _flashinfer_fused_add_rms_norm_nvfp4_quant(
|
||||
result: torch.Tensor,
|
||||
result_block_scale: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
block_scale_unswizzled: torch.Tensor,
|
||||
is_sf_swizzled_layout: bool,
|
||||
epsilon: float,
|
||||
) -> None:
|
||||
"""FlashInfer fused add + RMSNorm + NVFP4 quantization."""
|
||||
assert _FLASHINFER_ADD_RMSNORM_FP4QUANT is not None
|
||||
_FLASHINFER_ADD_RMSNORM_FP4QUANT(
|
||||
input,
|
||||
residual,
|
||||
weight,
|
||||
y_fp4=result.view(torch.float4_e2m1fn_x2),
|
||||
block_scale=result_block_scale.view(torch.float8_e4m3fn),
|
||||
global_scale=input_global_scale.reshape(1),
|
||||
eps=epsilon,
|
||||
block_size=16,
|
||||
scale_format="e4m3",
|
||||
is_sf_swizzled_layout=is_sf_swizzled_layout,
|
||||
output_both_sf_layouts=False,
|
||||
block_scale_unswizzled=block_scale_unswizzled,
|
||||
)
|
||||
|
||||
|
||||
def _flashinfer_fused_add_rms_norm_nvfp4_quant_fake(
|
||||
result: torch.Tensor,
|
||||
result_block_scale: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
block_scale_unswizzled: torch.Tensor,
|
||||
is_sf_swizzled_layout: bool,
|
||||
epsilon: float,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
_FLASHINFER_NVFP4_RMS_QUANT_OP: OpOverload | None = None
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and hasattr(torch, "float4_e2m1fn_x2")
|
||||
and has_flashinfer()
|
||||
):
|
||||
try:
|
||||
from flashinfer import add_rmsnorm_fp4quant
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
_FLASHINFER_ADD_RMSNORM_FP4QUANT = add_rmsnorm_fp4quant
|
||||
direct_register_custom_op(
|
||||
op_name="flashinfer_fused_add_rms_norm_nvfp4_quant",
|
||||
op_func=_flashinfer_fused_add_rms_norm_nvfp4_quant,
|
||||
mutates_args=["result", "result_block_scale", "residual"],
|
||||
fake_impl=_flashinfer_fused_add_rms_norm_nvfp4_quant_fake,
|
||||
)
|
||||
_FLASHINFER_NVFP4_RMS_QUANT_OP = (
|
||||
torch.ops.vllm.flashinfer_fused_add_rms_norm_nvfp4_quant.default
|
||||
)
|
||||
|
||||
|
||||
# TODO: extend rmsnorm quant kernels to support mixed input/weight dtypes,
|
||||
@@ -610,6 +679,97 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
|
||||
)
|
||||
|
||||
|
||||
class FusedAddRMSNormNvfp4QuantPattern:
|
||||
"""Fuse add-RMSNorm with NVFP4 quantization for either scale layout."""
|
||||
|
||||
def __init__(self, epsilon: float, is_sf_swizzled_layout: bool) -> None:
|
||||
assert _FLASHINFER_NVFP4_RMS_QUANT_OP is not None
|
||||
self.epsilon = epsilon
|
||||
self.is_sf_swizzled_layout = is_sf_swizzled_layout
|
||||
self.FUSED_OP = _FLASHINFER_NVFP4_RMS_QUANT_OP
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
def pattern(
|
||||
result: torch.Tensor,
|
||||
result_block_scale: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result_rms, updated_residual = vllm.ir.ops.fused_add_rms_norm(
|
||||
input, residual, weight, self.epsilon
|
||||
)
|
||||
at = auto_functionalized(
|
||||
torch.ops._C.scaled_fp4_quant.out,
|
||||
input=result_rms,
|
||||
input_scale=input_global_scale,
|
||||
is_sf_swizzled_layout=self.is_sf_swizzled_layout,
|
||||
output=result,
|
||||
output_scale=result_block_scale,
|
||||
)
|
||||
return at[1], updated_residual, at[2]
|
||||
|
||||
def replacement(
|
||||
result: torch.Tensor,
|
||||
result_block_scale: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hidden_size = input.shape[-1]
|
||||
num_tokens = input.numel() // hidden_size
|
||||
# This full-size dummy is required by FlashInfer's TVM-FFI tensor
|
||||
# validation even though output_both_sf_layouts=False leaves it untouched.
|
||||
block_scale_unswizzled = torch.empty(
|
||||
(num_tokens, hidden_size // 16),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=input.device,
|
||||
)
|
||||
at = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
result=result,
|
||||
result_block_scale=result_block_scale,
|
||||
residual=residual,
|
||||
input=input,
|
||||
weight=weight,
|
||||
input_global_scale=input_global_scale,
|
||||
block_scale_unswizzled=block_scale_unswizzled,
|
||||
is_sf_swizzled_layout=self.is_sf_swizzled_layout,
|
||||
epsilon=self.epsilon,
|
||||
)
|
||||
# result, updated residual, block scale in the requested layout
|
||||
return at[1], at[3], at[2]
|
||||
|
||||
inputs = [
|
||||
torch.empty(
|
||||
(5, 32), dtype=torch.uint8, device=current_platform.device_type
|
||||
),
|
||||
(
|
||||
empty_i32(128, 4)
|
||||
if self.is_sf_swizzled_layout
|
||||
else torch.empty(
|
||||
(5, 4),
|
||||
dtype=torch.uint8,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
),
|
||||
empty_bf16(5, 64),
|
||||
empty_bf16(64),
|
||||
empty_bf16(5, 64),
|
||||
empty_fp32(1),
|
||||
]
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
inputs,
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
class RMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses rms_norm & quant custom ops into a fused rms_norm_quant op.
|
||||
@@ -627,6 +787,14 @@ class RMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
# Make sure fused add patterns are before simple rms norm,
|
||||
# as the latter is a subset of the former in torch ops
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
if _FLASHINFER_NVFP4_RMS_QUANT_OP is not None and (
|
||||
current_platform.has_device_capability(100)
|
||||
):
|
||||
for is_sf_swizzled_layout in (True, False):
|
||||
FusedAddRMSNormNvfp4QuantPattern(
|
||||
epsilon, is_sf_swizzled_layout
|
||||
).register(self.patterns)
|
||||
|
||||
# Fuse fused_add_rms_norm + static fp8 quant
|
||||
FusedAddRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(
|
||||
self.patterns
|
||||
@@ -685,4 +853,5 @@ class RMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
FusedAddRMSNormStaticQuantPattern,
|
||||
FusedAddRMSNormDynamicQuantPattern,
|
||||
FusedAddRMSNormGroupQuantPattern,
|
||||
FusedAddRMSNormNvfp4QuantPattern,
|
||||
)
|
||||
|
||||
@@ -124,6 +124,20 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
5: "scale_out",
|
||||
}
|
||||
self.defunctionalize(graph, node, mutated_args)
|
||||
elif (
|
||||
hasattr(
|
||||
torch.ops.vllm,
|
||||
"flashinfer_fused_add_rms_norm_nvfp4_quant",
|
||||
)
|
||||
and at_target
|
||||
== torch.ops.vllm.flashinfer_fused_add_rms_norm_nvfp4_quant.default
|
||||
):
|
||||
mutated_args = {
|
||||
1: "result",
|
||||
2: "result_block_scale",
|
||||
3: "residual",
|
||||
}
|
||||
self.defunctionalize(graph, node, mutated_args)
|
||||
# For some reason we need to specify the args for both
|
||||
# silu_and_mul and silu_and_mul_quant. The kwargs
|
||||
# pathway gets the wrong answer.
|
||||
|
||||
Reference in New Issue
Block a user